[ES6] Add DFG/FTL support for accessor put operations
[WebKit-https.git] / Source / JavaScriptCore / jit / JITInlines.h
1 /*
2  * Copyright (C) 2008, 2012, 2013, 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #ifndef JITInlines_h
27 #define JITInlines_h
28
29 #if ENABLE(JIT)
30
31 #include "JSCInlines.h"
32
33 namespace JSC {
34
35 #if USE(JSVALUE64)
36 inline MacroAssembler::JumpList JIT::emitDoubleGetByVal(Instruction* instruction, PatchableJump& badType)
37 {
38     JumpList slowCases = emitDoubleLoad(instruction, badType);
39     moveDoubleTo64(fpRegT0, regT0);
40     sub64(tagTypeNumberRegister, regT0);
41     return slowCases;
42 }
43 #else
44 inline MacroAssembler::JumpList JIT::emitDoubleGetByVal(Instruction* instruction, PatchableJump& badType)
45 {
46     JumpList slowCases = emitDoubleLoad(instruction, badType);
47     moveDoubleToInts(fpRegT0, regT0, regT1);
48     return slowCases;
49 }
50 #endif // USE(JSVALUE64)
51
52 ALWAYS_INLINE MacroAssembler::JumpList JIT::emitLoadForArrayMode(Instruction* currentInstruction, JITArrayMode arrayMode, PatchableJump& badType)
53 {
54     switch (arrayMode) {
55     case JITInt32:
56         return emitInt32Load(currentInstruction, badType);
57     case JITDouble:
58         return emitDoubleLoad(currentInstruction, badType);
59     case JITContiguous:
60         return emitContiguousLoad(currentInstruction, badType);
61     case JITArrayStorage:
62         return emitArrayStorageLoad(currentInstruction, badType);
63     default:
64         break;
65     }
66     RELEASE_ASSERT_NOT_REACHED();
67     return MacroAssembler::JumpList();
68 }
69
70 inline MacroAssembler::JumpList JIT::emitContiguousGetByVal(Instruction* instruction, PatchableJump& badType, IndexingType expectedShape)
71 {
72     return emitContiguousLoad(instruction, badType, expectedShape);
73 }
74
75 inline MacroAssembler::JumpList JIT::emitArrayStorageGetByVal(Instruction* instruction, PatchableJump& badType)
76 {
77     return emitArrayStorageLoad(instruction, badType);
78 }
79
80 ALWAYS_INLINE bool JIT::isOperandConstantDouble(int src)
81 {
82     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isDouble();
83 }
84
85 ALWAYS_INLINE JSValue JIT::getConstantOperand(int src)
86 {
87     ASSERT(m_codeBlock->isConstantRegisterIndex(src));
88     return m_codeBlock->getConstant(src);
89 }
90
91 ALWAYS_INLINE void JIT::emitPutIntToCallFrameHeader(RegisterID from, JSStack::CallFrameHeaderEntry entry)
92 {
93 #if USE(JSVALUE32_64)
94     store32(TrustedImm32(Int32Tag), intTagFor(entry, callFrameRegister));
95     store32(from, intPayloadFor(entry, callFrameRegister));
96 #else
97     store64(from, addressFor(entry, callFrameRegister));
98 #endif
99 }
100
101 ALWAYS_INLINE void JIT::emitLoadCharacterString(RegisterID src, RegisterID dst, JumpList& failures)
102 {
103     failures.append(branchStructure(NotEqual, Address(src, JSCell::structureIDOffset()), m_vm->stringStructure.get()));
104     failures.append(branch32(NotEqual, MacroAssembler::Address(src, ThunkHelpers::jsStringLengthOffset()), TrustedImm32(1)));
105     loadPtr(MacroAssembler::Address(src, ThunkHelpers::jsStringValueOffset()), dst);
106     failures.append(branchTest32(Zero, dst));
107     loadPtr(MacroAssembler::Address(dst, StringImpl::flagsOffset()), regT1);
108     loadPtr(MacroAssembler::Address(dst, StringImpl::dataOffset()), dst);
109
110     JumpList is16Bit;
111     JumpList cont8Bit;
112     is16Bit.append(branchTest32(Zero, regT1, TrustedImm32(StringImpl::flagIs8Bit())));
113     load8(MacroAssembler::Address(dst, 0), dst);
114     cont8Bit.append(jump());
115     is16Bit.link(this);
116     load16(MacroAssembler::Address(dst, 0), dst);
117     cont8Bit.link(this);
118 }
119
120 ALWAYS_INLINE JIT::Call JIT::emitNakedCall(CodePtr function)
121 {
122     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
123     Call nakedCall = nearCall();
124     m_calls.append(CallRecord(nakedCall, m_bytecodeOffset, function.executableAddress()));
125     return nakedCall;
126 }
127
128 ALWAYS_INLINE JIT::Call JIT::emitNakedTailCall(CodePtr function)
129 {
130     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
131     Call nakedCall = nearTailCall();
132     m_calls.append(CallRecord(nakedCall, m_bytecodeOffset, function.executableAddress()));
133     return nakedCall;
134 }
135
136 ALWAYS_INLINE void JIT::updateTopCallFrame()
137 {
138     ASSERT(static_cast<int>(m_bytecodeOffset) >= 0);
139 #if USE(JSVALUE32_64)
140     Instruction* instruction = m_codeBlock->instructions().begin() + m_bytecodeOffset + 1; 
141     uint32_t locationBits = CallSiteIndex(instruction).bits();
142 #else
143     uint32_t locationBits = CallSiteIndex(m_bytecodeOffset + 1).bits();
144 #endif
145     store32(TrustedImm32(locationBits), intTagFor(JSStack::ArgumentCount));
146     storePtr(callFrameRegister, &m_vm->topCallFrame);
147 }
148
149 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheck(const FunctionPtr& function)
150 {
151     updateTopCallFrame();
152     MacroAssembler::Call call = appendCall(function);
153     exceptionCheck();
154     return call;
155 }
156
157 #if OS(WINDOWS) && CPU(X86_64)
158 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckAndSlowPathReturnType(const FunctionPtr& function)
159 {
160     updateTopCallFrame();
161     MacroAssembler::Call call = appendCallWithSlowPathReturnType(function);
162     exceptionCheck();
163     return call;
164 }
165 #endif
166
167 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithCallFrameRollbackOnException(const FunctionPtr& function)
168 {
169     updateTopCallFrame(); // The callee is responsible for setting topCallFrame to their caller
170     MacroAssembler::Call call = appendCall(function);
171     exceptionCheckWithCallFrameRollback();
172     return call;
173 }
174
175 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckSetJSValueResult(const FunctionPtr& function, int dst)
176 {
177     MacroAssembler::Call call = appendCallWithExceptionCheck(function);
178 #if USE(JSVALUE64)
179     emitPutVirtualRegister(dst, returnValueGPR);
180 #else
181     emitStore(dst, returnValueGPR2, returnValueGPR);
182 #endif
183     return call;
184 }
185
186 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckSetJSValueResultWithProfile(const FunctionPtr& function, int dst)
187 {
188     MacroAssembler::Call call = appendCallWithExceptionCheck(function);
189     emitValueProfilingSite();
190 #if USE(JSVALUE64)
191     emitPutVirtualRegister(dst, returnValueGPR);
192 #else
193     emitStore(dst, returnValueGPR2, returnValueGPR);
194 #endif
195     return call;
196 }
197
198 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_E operation)
199 {
200     setupArgumentsExecState();
201     return appendCallWithExceptionCheck(operation);
202 }
203
204 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(Z_JITOperation_E operation)
205 {
206     setupArgumentsExecState();
207     updateTopCallFrame();
208     return appendCall(operation);
209 }
210
211 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_E operation)
212 {
213     setupArgumentsExecState();
214     return appendCallWithExceptionCheck(operation);
215 }
216
217 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EJsc operation, GPRReg arg1)
218 {
219     setupArgumentsWithExecState(arg1);
220     return appendCallWithExceptionCheck(operation);
221 }
222
223 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EJscZ operation, GPRReg arg1, int32_t arg2)
224 {
225     setupArgumentsWithExecState(arg1, TrustedImm32(arg2));
226     return appendCallWithExceptionCheck(operation);
227 }
228
229 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EL operation, GPRReg arg1)
230 {
231     setupArgumentsWithExecState(arg1);
232     return appendCallWithExceptionCheck(operation);
233 }
234     
235 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EL operation, TrustedImmPtr arg1)
236 {
237     setupArgumentsWithExecState(arg1);
238     return appendCallWithExceptionCheck(operation);
239 }
240     
241 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EO operation, GPRReg arg)
242 {
243     setupArgumentsWithExecState(arg);
244     return appendCallWithExceptionCheck(operation);
245 }
246
247 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_ESt operation, Structure* structure)
248 {
249     setupArgumentsWithExecState(TrustedImmPtr(structure));
250     return appendCallWithExceptionCheck(operation);
251 }
252
253 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EZ operation, int32_t arg)
254 {
255     setupArgumentsWithExecState(TrustedImm32(arg));
256     return appendCallWithExceptionCheck(operation);
257 }
258
259 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_E operation, int dst)
260 {
261     setupArgumentsExecState();
262     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
263 }
264
265 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJcpZ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2, int32_t arg3)
266 {
267     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2, TrustedImm32(arg3));
268     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
269 }
270
271 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJcpZ operation, int dst, ArrayAllocationProfile* arg1, const JSValue* arg2, int32_t arg3)
272 {
273     setupArgumentsWithExecState(TrustedImmPtr(arg1), TrustedImmPtr(arg2), TrustedImm32(arg3));
274     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
275 }
276
277 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EC operation, int dst, JSCell* cell)
278 {
279     setupArgumentsWithExecState(TrustedImmPtr(cell));
280     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
281 }
282
283 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EC operation, JSCell* cell)
284 {
285     setupArgumentsWithExecState(TrustedImmPtr(cell));
286     return appendCallWithExceptionCheck(operation);
287 }
288
289 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscC operation, int dst, GPRReg arg1, JSCell* cell)
290 {
291     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell));
292     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
293 }
294
295 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscCJ operation, int dst, GPRReg arg1, JSCell* cell, GPRReg arg2)
296 {
297     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell), arg2);
298     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
299 }
300     
301 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EP operation, int dst, void* pointer)
302 {
303     setupArgumentsWithExecState(TrustedImmPtr(pointer));
304     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
305 }
306
307 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(WithProfileTag, J_JITOperation_EPc operation, int dst, Instruction* bytecodePC)
308 {
309     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
310     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
311 }
312
313 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EPc operation, int dst, Instruction* bytecodePC)
314 {
315     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
316     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
317 }
318
319 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZ operation, int dst, int32_t arg)
320 {
321     setupArgumentsWithExecState(TrustedImm32(arg));
322     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
323 }
324
325 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZZ operation, int dst, int32_t arg1, int32_t arg2)
326 {
327     setupArgumentsWithExecState(TrustedImm32(arg1), TrustedImm32(arg2));
328     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
329 }
330
331 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_ECC operation, RegisterID regOp1, RegisterID regOp2)
332 {
333     setupArgumentsWithExecState(regOp1, regOp2);
334     return appendCallWithExceptionCheck(operation);
335 }
336
337 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EOJss operation, RegisterID regOp1, RegisterID regOp2)
338 {
339     setupArgumentsWithExecState(regOp1, regOp2);
340     return appendCallWithExceptionCheck(operation);
341 }
342
343 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Sprt_JITOperation_EZ operation, int32_t op)
344 {
345 #if OS(WINDOWS) && CPU(X86_64)
346     setupArgumentsWithExecStateForCallWithSlowPathReturnType(TrustedImm32(op));
347     return appendCallWithExceptionCheckAndSlowPathReturnType(operation);
348 #else
349     setupArgumentsWithExecState(TrustedImm32(op));
350     return appendCallWithExceptionCheck(operation);
351 #endif
352 }
353
354 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_E operation)
355 {
356     setupArgumentsExecState();
357     return appendCallWithExceptionCheck(operation);
358 }
359
360 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EC operation, RegisterID regOp)
361 {
362     setupArgumentsWithExecState(regOp);
363     return appendCallWithExceptionCheck(operation);
364 }
365
366 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECC operation, RegisterID regOp1, RegisterID regOp2)
367 {
368     setupArgumentsWithExecState(regOp1, regOp2);
369     return appendCallWithExceptionCheck(operation);
370 }
371
372 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EE operation, RegisterID regOp)
373 {
374     setupArgumentsWithExecState(regOp);
375     updateTopCallFrame();
376     return appendCallWithExceptionCheck(operation);
377 }
378
379 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EPc operation, Instruction* bytecodePC)
380 {
381     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
382     return appendCallWithExceptionCheck(operation);
383 }
384
385 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZ operation, int32_t op)
386 {
387     setupArgumentsWithExecState(TrustedImm32(op));
388     return appendCallWithExceptionCheck(operation);
389 }
390
391 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(J_JITOperation_E operation)
392 {
393     setupArgumentsExecState();
394     return appendCallWithCallFrameRollbackOnException(operation);
395 }
396
397 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(V_JITOperation_ECb operation, CodeBlock* pointer)
398 {
399     setupArgumentsWithExecState(TrustedImmPtr(pointer));
400     return appendCallWithCallFrameRollbackOnException(operation);
401 }
402
403 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(Z_JITOperation_E operation)
404 {
405     setupArgumentsExecState();
406     return appendCallWithCallFrameRollbackOnException(operation);
407 }
408
409 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZC operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4)
410 {
411     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4);
412     return appendCallWithExceptionCheck(operation);
413 }
414
415 #if USE(JSVALUE64)
416 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJZZ operation, GPRReg arg1, int32_t arg2, int32_t arg3)
417 {
418     setupArgumentsWithExecState(arg1, TrustedImm32(arg2), TrustedImm32(arg3));
419     return appendCallWithExceptionCheck(operation);
420 }
421
422 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(F_JITOperation_EFJZZ operation, GPRReg arg1, GPRReg arg2, int32_t arg3, GPRReg arg4)
423 {
424     setupArgumentsWithExecState(arg1, arg2, TrustedImm32(arg3), arg4);
425     return appendCallWithExceptionCheck(operation);
426 }
427
428 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, RegisterID regOp1, RegisterID regOp2, UniquedStringImpl* uid)
429 {
430     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), regOp1, regOp2, TrustedImmPtr(uid));
431     return appendCallWithExceptionCheck(operation);
432 }
433
434 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJ operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3)
435 {
436     setupArgumentsWithExecState(regOp1, regOp2, regOp3);
437     return appendCallWithExceptionCheck(operation);
438 }
439
440 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJAp operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3, ArrayProfile* arrayProfile)
441 {
442     setupArgumentsWithExecState(regOp1, regOp2, regOp3, TrustedImmPtr(arrayProfile));
443     return appendCallWithExceptionCheck(operation);
444 }
445
446 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJBy operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3, ByValInfo* byValInfo)
447 {
448     setupArgumentsWithExecState(regOp1, regOp2, regOp3, TrustedImmPtr(byValInfo));
449     return appendCallWithExceptionCheck(operation);
450 }
451
452 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZJ operation, int dst, GPRReg arg)
453 {
454     setupArgumentsWithExecState(TrustedImm32(dst), arg);
455     return appendCallWithExceptionCheck(operation);
456 }
457
458 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1, UniquedStringImpl* uid)
459 {
460     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
461     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
462 }
463
464 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_EJJ operation, int dst, GPRReg arg1, GPRReg arg2)
465 {
466     setupArgumentsWithExecState(arg1, arg2);
467     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
468 }
469
470 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2)
471 {
472     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2);
473     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
474 }
475
476 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJ operation, int dst, GPRReg arg1)
477 {
478     setupArgumentsWithExecState(arg1);
479     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
480 }
481
482 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJIdc operation, int dst, GPRReg arg1, const Identifier* arg2)
483 {
484     setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
485     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
486 }
487
488 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJ operation, int dst, GPRReg arg1, GPRReg arg2)
489 {
490     setupArgumentsWithExecState(arg1, arg2);
491     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
492 }
493
494 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJAp operation, int dst, GPRReg arg1, GPRReg arg2, ArrayProfile* arrayProfile)
495 {
496     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(arrayProfile));
497     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
498 }
499
500 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJBy operation, int dst, GPRReg arg1, GPRReg arg2, ByValInfo* byValInfo)
501 {
502     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(byValInfo));
503     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
504 }
505
506 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(V_JITOperation_EJ operation, GPRReg arg1)
507 {
508     setupArgumentsWithExecState(arg1);
509     updateTopCallFrame();
510     return appendCall(operation);
511 }
512
513 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_EJS operation, GPRReg arg1, size_t arg2)
514 {
515     setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
516     return appendCallWithExceptionCheck(operation);
517 }
518
519 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJ operation, RegisterID regOp)
520 {
521     setupArgumentsWithExecState(regOp);
522     return appendCallWithExceptionCheck(operation);
523 }
524
525 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJJ operation, RegisterID regOp1, RegisterID regOp2)
526 {
527     setupArgumentsWithExecState(regOp1, regOp2);
528     return appendCallWithExceptionCheck(operation);
529 }
530
531 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZSymtabJ operation, int op1, SymbolTable* symbolTable, RegisterID regOp3)
532 {
533     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), regOp3);
534     return appendCallWithExceptionCheck(operation);
535 }
536
537 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZSymtabJ operation, int op1, SymbolTable* symbolTable, RegisterID regOp3)
538 {
539     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), regOp3);
540     return appendCallWithExceptionCheck(operation);
541 }
542
543 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJ operation, RegisterID regOp)
544 {
545     setupArgumentsWithExecState(regOp);
546     return appendCallWithExceptionCheck(operation);
547 }
548
549 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZJJ operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4, RegisterID regOp5)
550 {
551     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4, regOp5);
552     return appendCallWithExceptionCheck(operation);
553 }
554
555 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJZC operation, RegisterID regOp1, RegisterID regOp2, int32_t op3, RegisterID regOp4)
556 {
557     setupArgumentsWithExecState(regOp1, regOp2, TrustedImm32(op3), regOp4);
558     return appendCallWithExceptionCheck(operation);
559 }
560
561 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZ operation, RegisterID regOp1, int32_t op2)
562 {
563     setupArgumentsWithExecState(regOp1, TrustedImm32(op2));
564     return appendCallWithExceptionCheck(operation);
565 }
566
567 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZJ operation, RegisterID regOp1, int32_t op2, RegisterID regOp3)
568 {
569     setupArgumentsWithExecState(regOp1, TrustedImm32(op2), regOp3);
570     return appendCallWithExceptionCheck(operation);
571 }
572
573 #else // USE(JSVALUE32_64)
574
575 // EncodedJSValue in JSVALUE32_64 is a 64-bit integer. When being compiled in ARM EABI, it must be aligned even-numbered register (r0, r2 or [sp]).
576 // To avoid assemblies from using wrong registers, let's occupy r1 or r3 with a dummy argument when necessary.
577 #if (COMPILER_SUPPORTS(EABI) && CPU(ARM)) || CPU(MIPS)
578 #define EABI_32BIT_DUMMY_ARG      TrustedImm32(0),
579 #else
580 #define EABI_32BIT_DUMMY_ARG
581 #endif
582
583 // JSVALUE32_64 is a 64-bit integer that cannot be put half in an argument register and half on stack when using SH4 architecture.
584 // To avoid this, let's occupy the 4th argument register (r7) with a dummy argument when necessary. This must only be done when there
585 // is no other 32-bit value argument behind this 64-bit JSValue.
586 #if CPU(SH4)
587 #define SH4_32BIT_DUMMY_ARG      TrustedImm32(0),
588 #else
589 #define SH4_32BIT_DUMMY_ARG
590 #endif
591
592 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(V_JITOperation_EJ operation, GPRReg arg1Tag, GPRReg arg1Payload)
593 {
594     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
595     updateTopCallFrame();
596     return appendCall(operation);
597 }
598
599 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJZZ operation, GPRReg arg1Tag, GPRReg arg1Payload, int32_t arg2, int32_t arg3)
600 {
601     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImm32(arg2), TrustedImm32(arg3));
602     return appendCallWithExceptionCheck(operation);
603 }
604
605 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(F_JITOperation_EFJZZ operation, GPRReg arg1, GPRReg arg2Tag, GPRReg arg2Payload, int32_t arg3, GPRReg arg4)
606 {
607     setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, TrustedImm32(arg3), arg4);
608     return appendCallWithExceptionCheck(operation);
609 }
610     
611 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2Tag, GPRReg arg2Payload)
612 {
613     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2Payload, arg2Tag);
614     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
615 }
616
617 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload)
618 {
619     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
620     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
621 }
622
623 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* uid)
624 {
625     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, arg1Tag, TrustedImmPtr(uid));
626     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
627 }
628
629 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJIdc operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, const Identifier* arg2)
630 {
631     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(arg2));
632     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
633 }
634
635 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
636 {
637     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
638     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
639 }
640
641 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJAp operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload, ArrayProfile* arrayProfile)
642 {
643     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag, TrustedImmPtr(arrayProfile));
644     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
645 }
646
647 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJBy operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload, ByValInfo* byValInfo)
648 {
649     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag, TrustedImmPtr(byValInfo));
650     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
651 }
652
653 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_EJJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
654 {
655     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
656     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
657 }
658
659 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_EJS operation, GPRReg arg1Tag, GPRReg arg1Payload, size_t arg2)
660 {
661     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(arg2));
662     return appendCallWithExceptionCheck(operation);
663 }
664
665 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJ operation, RegisterID argTag, RegisterID argPayload)
666 {
667     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG argPayload, argTag);
668     return appendCallWithExceptionCheck(operation);
669 }
670
671 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJJ operation, RegisterID arg1Tag, RegisterID arg1Payload, RegisterID arg2Tag, RegisterID arg2Payload)
672 {
673     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
674     return appendCallWithExceptionCheck(operation);
675 }
676
677 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZCC operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4, RegisterID regOp5)
678 {
679     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4, regOp5);
680     return appendCallWithExceptionCheck(operation);
681 }
682
683 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJZC operation, RegisterID arg1, RegisterID arg2Tag, RegisterID arg2Payload, int32_t arg3, RegisterID arg4)
684 {
685     setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, TrustedImm32(arg3), arg4);
686     return appendCallWithExceptionCheck(operation);
687 }
688
689 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload)
690 {
691     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag);
692     return appendCallWithExceptionCheck(operation);
693 }
694
695 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZSymtabJ operation, int32_t op1, SymbolTable* symbolTable, RegisterID regOp3Tag, RegisterID regOp3Payload)
696 {
697     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), EABI_32BIT_DUMMY_ARG regOp3Payload, regOp3Tag);
698     return appendCallWithExceptionCheck(operation);
699 }
700
701 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, UniquedStringImpl* uid)
702 {
703     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), regOp1Payload, regOp1Tag, regOp2Payload, regOp2Tag, TrustedImmPtr(uid));
704     return appendCallWithExceptionCheck(operation);
705 }
706
707 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload)
708 {
709     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag);
710     return appendCallWithExceptionCheck(operation);
711 }
712
713 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJAp operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload, ArrayProfile* arrayProfile)
714 {
715     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag, TrustedImmPtr(arrayProfile));
716     return appendCallWithExceptionCheck(operation);
717 }
718
719 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJBy operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload, ByValInfo* byValInfo)
720 {
721     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag, TrustedImmPtr(byValInfo));
722     return appendCallWithExceptionCheck(operation);
723 }
724
725 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZJ operation, int dst, RegisterID regOp1Tag, RegisterID regOp1Payload)
726 {
727     setupArgumentsWithExecState(TrustedImm32(dst), regOp1Payload, regOp1Tag);
728     return appendCallWithExceptionCheck(operation);
729 }
730
731 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, int32_t op2)
732 {
733     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, TrustedImm32(op2));
734     return appendCallWithExceptionCheck(operation);
735 }
736
737 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, int32_t op2, RegisterID regOp3Tag, RegisterID regOp3Payload)
738 {
739     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, TrustedImm32(op2), EABI_32BIT_DUMMY_ARG regOp3Payload, regOp3Tag);
740     return appendCallWithExceptionCheck(operation);
741 }
742     
743 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscCJ operation, int dst, GPRReg arg1, JSCell* cell, GPRReg arg2Tag, GPRReg arg2Payload)
744 {
745     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell), EABI_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
746     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
747 }
748
749 #undef EABI_32BIT_DUMMY_ARG
750 #undef SH4_32BIT_DUMMY_ARG
751
752 #endif // USE(JSVALUE32_64)
753
754 ALWAYS_INLINE JIT::Jump JIT::checkStructure(RegisterID reg, Structure* structure)
755 {
756     return branchStructure(NotEqual, Address(reg, JSCell::structureIDOffset()), structure);
757 }
758
759 ALWAYS_INLINE void JIT::linkSlowCaseIfNotJSCell(Vector<SlowCaseEntry>::iterator& iter, int vReg)
760 {
761     if (!m_codeBlock->isKnownNotImmediate(vReg))
762         linkSlowCase(iter);
763 }
764
765 ALWAYS_INLINE void JIT::linkAllSlowCasesForBytecodeOffset(Vector<SlowCaseEntry>& slowCases, Vector<SlowCaseEntry>::iterator& iter, unsigned bytecodeOffset)
766 {
767     while (iter != slowCases.end() && iter->to == bytecodeOffset) {
768         iter->from.link(this);
769         ++iter;
770     }
771 }
772
773 ALWAYS_INLINE void JIT::addSlowCase(Jump jump)
774 {
775     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
776
777     m_slowCases.append(SlowCaseEntry(jump, m_bytecodeOffset));
778 }
779
780 ALWAYS_INLINE void JIT::addSlowCase(JumpList jumpList)
781 {
782     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
783
784     const JumpList::JumpVector& jumpVector = jumpList.jumps();
785     size_t size = jumpVector.size();
786     for (size_t i = 0; i < size; ++i)
787         m_slowCases.append(SlowCaseEntry(jumpVector[i], m_bytecodeOffset));
788 }
789
790 ALWAYS_INLINE void JIT::addSlowCase()
791 {
792     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
793     
794     Jump emptyJump; // Doing it this way to make Windows happy.
795     m_slowCases.append(SlowCaseEntry(emptyJump, m_bytecodeOffset));
796 }
797
798 ALWAYS_INLINE void JIT::addJump(Jump jump, int relativeOffset)
799 {
800     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
801
802     m_jmpTable.append(JumpTable(jump, m_bytecodeOffset + relativeOffset));
803 }
804
805 ALWAYS_INLINE void JIT::emitJumpSlowToHot(Jump jump, int relativeOffset)
806 {
807     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
808
809     jump.linkTo(m_labels[m_bytecodeOffset + relativeOffset], this);
810 }
811
812 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfCellObject(RegisterID cellReg)
813 {
814     return branch8(AboveOrEqual, Address(cellReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ObjectType));
815 }
816
817 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfCellNotObject(RegisterID cellReg)
818 {
819     return branch8(Below, Address(cellReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ObjectType));
820 }
821
822 #if ENABLE(SAMPLING_FLAGS)
823 ALWAYS_INLINE void JIT::setSamplingFlag(int32_t flag)
824 {
825     ASSERT(flag >= 1);
826     ASSERT(flag <= 32);
827     or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
828 }
829
830 ALWAYS_INLINE void JIT::clearSamplingFlag(int32_t flag)
831 {
832     ASSERT(flag >= 1);
833     ASSERT(flag <= 32);
834     and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
835 }
836 #endif
837
838 #if ENABLE(SAMPLING_COUNTERS)
839 ALWAYS_INLINE void JIT::emitCount(AbstractSamplingCounter& counter, int32_t count)
840 {
841     add64(TrustedImm32(count), AbsoluteAddress(counter.addressOfCounter()));
842 }
843 #endif
844
845 #if ENABLE(OPCODE_SAMPLING)
846 #if CPU(X86_64)
847 ALWAYS_INLINE void JIT::sampleInstruction(Instruction* instruction, bool inHostFunction)
848 {
849     move(TrustedImmPtr(m_interpreter->sampler()->sampleSlot()), X86Registers::ecx);
850     storePtr(TrustedImmPtr(m_interpreter->sampler()->encodeSample(instruction, inHostFunction)), X86Registers::ecx);
851 }
852 #else
853 ALWAYS_INLINE void JIT::sampleInstruction(Instruction* instruction, bool inHostFunction)
854 {
855     storePtr(TrustedImmPtr(m_interpreter->sampler()->encodeSample(instruction, inHostFunction)), m_interpreter->sampler()->sampleSlot());
856 }
857 #endif
858 #endif
859
860 #if ENABLE(CODEBLOCK_SAMPLING)
861 #if CPU(X86_64)
862 ALWAYS_INLINE void JIT::sampleCodeBlock(CodeBlock* codeBlock)
863 {
864     move(TrustedImmPtr(m_interpreter->sampler()->codeBlockSlot()), X86Registers::ecx);
865     storePtr(TrustedImmPtr(codeBlock), X86Registers::ecx);
866 }
867 #else
868 ALWAYS_INLINE void JIT::sampleCodeBlock(CodeBlock* codeBlock)
869 {
870     storePtr(TrustedImmPtr(codeBlock), m_interpreter->sampler()->codeBlockSlot());
871 }
872 #endif
873 #endif
874
875 ALWAYS_INLINE bool JIT::isOperandConstantChar(int src)
876 {
877     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isString() && asString(getConstantOperand(src).asCell())->length() == 1;
878 }
879
880 template<typename StructureType>
881 inline void JIT::emitAllocateJSObject(RegisterID allocator, StructureType structure, RegisterID result, RegisterID scratch)
882 {
883     loadPtr(Address(allocator, MarkedAllocator::offsetOfFreeListHead()), result);
884     addSlowCase(branchTestPtr(Zero, result));
885
886     // remove the object from the free list
887     loadPtr(Address(result), scratch);
888     storePtr(scratch, Address(allocator, MarkedAllocator::offsetOfFreeListHead()));
889
890     // initialize the object's property storage pointer
891     storePtr(TrustedImmPtr(0), Address(result, JSObject::butterflyOffset()));
892
893     // initialize the object's structure
894     emitStoreStructureWithTypeInfo(structure, result, scratch);
895 }
896
897 inline void JIT::emitValueProfilingSite(ValueProfile* valueProfile)
898 {
899     ASSERT(shouldEmitProfiling());
900     ASSERT(valueProfile);
901
902     const RegisterID value = regT0;
903 #if USE(JSVALUE32_64)
904     const RegisterID valueTag = regT1;
905 #endif
906     
907     // We're in a simple configuration: only one bucket, so we can just do a direct
908     // store.
909 #if USE(JSVALUE64)
910     store64(value, valueProfile->m_buckets);
911 #else
912     EncodedValueDescriptor* descriptor = bitwise_cast<EncodedValueDescriptor*>(valueProfile->m_buckets);
913     store32(value, &descriptor->asBits.payload);
914     store32(valueTag, &descriptor->asBits.tag);
915 #endif
916 }
917
918 inline void JIT::emitValueProfilingSite(unsigned bytecodeOffset)
919 {
920     if (!shouldEmitProfiling())
921         return;
922     emitValueProfilingSite(m_codeBlock->valueProfileForBytecodeOffset(bytecodeOffset));
923 }
924
925 inline void JIT::emitValueProfilingSite()
926 {
927     emitValueProfilingSite(m_bytecodeOffset);
928 }
929
930 inline void JIT::emitArrayProfilingSiteWithCell(RegisterID cell, RegisterID indexingType, ArrayProfile* arrayProfile)
931 {
932     if (shouldEmitProfiling()) {
933         load32(MacroAssembler::Address(cell, JSCell::structureIDOffset()), indexingType);
934         store32(indexingType, arrayProfile->addressOfLastSeenStructureID());
935     }
936
937     load8(Address(cell, JSCell::indexingTypeOffset()), indexingType);
938 }
939
940 inline void JIT::emitArrayProfilingSiteForBytecodeIndexWithCell(RegisterID cell, RegisterID indexingType, unsigned bytecodeIndex)
941 {
942     emitArrayProfilingSiteWithCell(cell, indexingType, m_codeBlock->getOrAddArrayProfile(bytecodeIndex));
943 }
944
945 inline void JIT::emitArrayProfileStoreToHoleSpecialCase(ArrayProfile* arrayProfile)
946 {
947     store8(TrustedImm32(1), arrayProfile->addressOfMayStoreToHole());
948 }
949
950 inline void JIT::emitArrayProfileOutOfBoundsSpecialCase(ArrayProfile* arrayProfile)
951 {
952     store8(TrustedImm32(1), arrayProfile->addressOfOutOfBounds());
953 }
954
955 static inline bool arrayProfileSaw(ArrayModes arrayModes, IndexingType capability)
956 {
957     return arrayModesInclude(arrayModes, capability);
958 }
959
960 inline JITArrayMode JIT::chooseArrayMode(ArrayProfile* profile)
961 {
962     ConcurrentJITLocker locker(m_codeBlock->m_lock);
963     profile->computeUpdatedPrediction(locker, m_codeBlock);
964     ArrayModes arrayModes = profile->observedArrayModes(locker);
965     if (arrayProfileSaw(arrayModes, DoubleShape))
966         return JITDouble;
967     if (arrayProfileSaw(arrayModes, Int32Shape))
968         return JITInt32;
969     if (arrayProfileSaw(arrayModes, ArrayStorageShape))
970         return JITArrayStorage;
971     return JITContiguous;
972 }
973
974 #if USE(JSVALUE32_64)
975
976 inline void JIT::emitLoadTag(int index, RegisterID tag)
977 {
978     if (m_codeBlock->isConstantRegisterIndex(index)) {
979         move(Imm32(getConstantOperand(index).tag()), tag);
980         return;
981     }
982
983     load32(tagFor(index), tag);
984 }
985
986 inline void JIT::emitLoadPayload(int index, RegisterID payload)
987 {
988     if (m_codeBlock->isConstantRegisterIndex(index)) {
989         move(Imm32(getConstantOperand(index).payload()), payload);
990         return;
991     }
992
993     load32(payloadFor(index), payload);
994 }
995
996 inline void JIT::emitLoad(const JSValue& v, RegisterID tag, RegisterID payload)
997 {
998     move(Imm32(v.payload()), payload);
999     move(Imm32(v.tag()), tag);
1000 }
1001
1002 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, JSValueRegs dst)
1003 {
1004     emitLoad(src, dst.tagGPR(), dst.payloadGPR());
1005 }
1006
1007 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, JSValueRegs from)
1008 {
1009     emitStore(dst, from.tagGPR(), from.payloadGPR());
1010 }
1011
1012 inline void JIT::emitLoad(int index, RegisterID tag, RegisterID payload, RegisterID base)
1013 {
1014     RELEASE_ASSERT(tag != payload);
1015
1016     if (base == callFrameRegister) {
1017         RELEASE_ASSERT(payload != base);
1018         emitLoadPayload(index, payload);
1019         emitLoadTag(index, tag);
1020         return;
1021     }
1022
1023     if (payload == base) { // avoid stomping base
1024         load32(tagFor(index, base), tag);
1025         load32(payloadFor(index, base), payload);
1026         return;
1027     }
1028
1029     load32(payloadFor(index, base), payload);
1030     load32(tagFor(index, base), tag);
1031 }
1032
1033 inline void JIT::emitLoad2(int index1, RegisterID tag1, RegisterID payload1, int index2, RegisterID tag2, RegisterID payload2)
1034 {
1035     emitLoad(index2, tag2, payload2);
1036     emitLoad(index1, tag1, payload1);
1037 }
1038
1039 inline void JIT::emitLoadDouble(int index, FPRegisterID value)
1040 {
1041     if (m_codeBlock->isConstantRegisterIndex(index)) {
1042         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1043         loadDouble(TrustedImmPtr(&inConstantPool), value);
1044     } else
1045         loadDouble(addressFor(index), value);
1046 }
1047
1048 inline void JIT::emitLoadInt32ToDouble(int index, FPRegisterID value)
1049 {
1050     if (m_codeBlock->isConstantRegisterIndex(index)) {
1051         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1052         char* bytePointer = reinterpret_cast<char*>(&inConstantPool);
1053         convertInt32ToDouble(AbsoluteAddress(bytePointer + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), value);
1054     } else
1055         convertInt32ToDouble(payloadFor(index), value);
1056 }
1057
1058 inline void JIT::emitStore(int index, RegisterID tag, RegisterID payload, RegisterID base)
1059 {
1060     store32(payload, payloadFor(index, base));
1061     store32(tag, tagFor(index, base));
1062 }
1063
1064 inline void JIT::emitStoreInt32(int index, RegisterID payload, bool indexIsInt32)
1065 {
1066     store32(payload, payloadFor(index, callFrameRegister));
1067     if (!indexIsInt32)
1068         store32(TrustedImm32(JSValue::Int32Tag), tagFor(index, callFrameRegister));
1069 }
1070
1071 inline void JIT::emitStoreInt32(int index, TrustedImm32 payload, bool indexIsInt32)
1072 {
1073     store32(payload, payloadFor(index, callFrameRegister));
1074     if (!indexIsInt32)
1075         store32(TrustedImm32(JSValue::Int32Tag), tagFor(index, callFrameRegister));
1076 }
1077
1078 inline void JIT::emitStoreCell(int index, RegisterID payload, bool indexIsCell)
1079 {
1080     store32(payload, payloadFor(index, callFrameRegister));
1081     if (!indexIsCell)
1082         store32(TrustedImm32(JSValue::CellTag), tagFor(index, callFrameRegister));
1083 }
1084
1085 inline void JIT::emitStoreBool(int index, RegisterID payload, bool indexIsBool)
1086 {
1087     store32(payload, payloadFor(index, callFrameRegister));
1088     if (!indexIsBool)
1089         store32(TrustedImm32(JSValue::BooleanTag), tagFor(index, callFrameRegister));
1090 }
1091
1092 inline void JIT::emitStoreDouble(int index, FPRegisterID value)
1093 {
1094     storeDouble(value, addressFor(index));
1095 }
1096
1097 inline void JIT::emitStore(int index, const JSValue constant, RegisterID base)
1098 {
1099     store32(Imm32(constant.payload()), payloadFor(index, base));
1100     store32(Imm32(constant.tag()), tagFor(index, base));
1101 }
1102
1103 ALWAYS_INLINE void JIT::emitInitRegister(int dst)
1104 {
1105     emitStore(dst, jsUndefined());
1106 }
1107
1108 inline void JIT::emitJumpSlowCaseIfNotJSCell(int virtualRegisterIndex)
1109 {
1110     if (!m_codeBlock->isKnownNotImmediate(virtualRegisterIndex)) {
1111         if (m_codeBlock->isConstantRegisterIndex(virtualRegisterIndex))
1112             addSlowCase(jump());
1113         else
1114             addSlowCase(emitJumpIfNotJSCell(virtualRegisterIndex));
1115     }
1116 }
1117
1118 inline void JIT::emitJumpSlowCaseIfNotJSCell(int virtualRegisterIndex, RegisterID tag)
1119 {
1120     if (!m_codeBlock->isKnownNotImmediate(virtualRegisterIndex)) {
1121         if (m_codeBlock->isConstantRegisterIndex(virtualRegisterIndex))
1122             addSlowCase(jump());
1123         else
1124             addSlowCase(branch32(NotEqual, tag, TrustedImm32(JSValue::CellTag)));
1125     }
1126 }
1127
1128 ALWAYS_INLINE bool JIT::isOperandConstantInt(int src)
1129 {
1130     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isInt32();
1131 }
1132
1133 ALWAYS_INLINE bool JIT::getOperandConstantInt(int op1, int op2, int& op, int32_t& constant)
1134 {
1135     if (isOperandConstantInt(op1)) {
1136         constant = getConstantOperand(op1).asInt32();
1137         op = op2;
1138         return true;
1139     }
1140
1141     if (isOperandConstantInt(op2)) {
1142         constant = getConstantOperand(op2).asInt32();
1143         op = op1;
1144         return true;
1145     }
1146     
1147     return false;
1148 }
1149
1150 #else // USE(JSVALUE32_64)
1151
1152 // get arg puts an arg from the SF register array into a h/w register
1153 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, RegisterID dst)
1154 {
1155     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
1156
1157     // TODO: we want to reuse values that are already in registers if we can - add a register allocator!
1158     if (m_codeBlock->isConstantRegisterIndex(src)) {
1159         JSValue value = m_codeBlock->getConstant(src);
1160         if (!value.isNumber())
1161             move(TrustedImm64(JSValue::encode(value)), dst);
1162         else
1163             move(Imm64(JSValue::encode(value)), dst);
1164         return;
1165     }
1166
1167     load64(Address(callFrameRegister, src * sizeof(Register)), dst);
1168 }
1169
1170 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, JSValueRegs dst)
1171 {
1172     emitGetVirtualRegister(src, dst.payloadGPR());
1173 }
1174
1175 ALWAYS_INLINE void JIT::emitGetVirtualRegister(VirtualRegister src, RegisterID dst)
1176 {
1177     emitGetVirtualRegister(src.offset(), dst);
1178 }
1179
1180 ALWAYS_INLINE void JIT::emitGetVirtualRegisters(int src1, RegisterID dst1, int src2, RegisterID dst2)
1181 {
1182     emitGetVirtualRegister(src1, dst1);
1183     emitGetVirtualRegister(src2, dst2);
1184 }
1185
1186 ALWAYS_INLINE void JIT::emitGetVirtualRegisters(VirtualRegister src1, RegisterID dst1, VirtualRegister src2, RegisterID dst2)
1187 {
1188     emitGetVirtualRegisters(src1.offset(), dst1, src2.offset(), dst2);
1189 }
1190
1191 ALWAYS_INLINE int32_t JIT::getOperandConstantInt(int src)
1192 {
1193     return getConstantOperand(src).asInt32();
1194 }
1195
1196 ALWAYS_INLINE bool JIT::isOperandConstantInt(int src)
1197 {
1198     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isInt32();
1199 }
1200
1201 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, RegisterID from)
1202 {
1203     store64(from, Address(callFrameRegister, dst * sizeof(Register)));
1204 }
1205
1206 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, JSValueRegs from)
1207 {
1208     emitPutVirtualRegister(dst, from.payloadGPR());
1209 }
1210
1211 ALWAYS_INLINE void JIT::emitPutVirtualRegister(VirtualRegister dst, RegisterID from)
1212 {
1213     emitPutVirtualRegister(dst.offset(), from);
1214 }
1215
1216 ALWAYS_INLINE void JIT::emitInitRegister(int dst)
1217 {
1218     store64(TrustedImm64(JSValue::encode(jsUndefined())), Address(callFrameRegister, dst * sizeof(Register)));
1219 }
1220
1221 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfJSCell(RegisterID reg)
1222 {
1223     return branchTest64(Zero, reg, tagMaskRegister);
1224 }
1225
1226 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfBothJSCells(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1227 {
1228     move(reg1, scratch);
1229     or64(reg2, scratch);
1230     return emitJumpIfJSCell(scratch);
1231 }
1232
1233 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfJSCell(RegisterID reg)
1234 {
1235     addSlowCase(emitJumpIfJSCell(reg));
1236 }
1237
1238 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotJSCell(RegisterID reg)
1239 {
1240     addSlowCase(emitJumpIfNotJSCell(reg));
1241 }
1242
1243 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotJSCell(RegisterID reg, int vReg)
1244 {
1245     if (!m_codeBlock->isKnownNotImmediate(vReg))
1246         emitJumpSlowCaseIfNotJSCell(reg);
1247 }
1248
1249 inline void JIT::emitLoadDouble(int index, FPRegisterID value)
1250 {
1251     if (m_codeBlock->isConstantRegisterIndex(index)) {
1252         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1253         loadDouble(TrustedImmPtr(&inConstantPool), value);
1254     } else
1255         loadDouble(addressFor(index), value);
1256 }
1257
1258 inline void JIT::emitLoadInt32ToDouble(int index, FPRegisterID value)
1259 {
1260     if (m_codeBlock->isConstantRegisterIndex(index)) {
1261         ASSERT(isOperandConstantInt(index));
1262         convertInt32ToDouble(Imm32(getConstantOperand(index).asInt32()), value);
1263     } else
1264         convertInt32ToDouble(addressFor(index), value);
1265 }
1266
1267 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfInt(RegisterID reg)
1268 {
1269     return branch64(AboveOrEqual, reg, tagTypeNumberRegister);
1270 }
1271
1272 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfNotInt(RegisterID reg)
1273 {
1274     return branch64(Below, reg, tagTypeNumberRegister);
1275 }
1276
1277 ALWAYS_INLINE JIT::PatchableJump JIT::emitPatchableJumpIfNotInt(RegisterID reg)
1278 {
1279     return patchableBranch64(Below, reg, tagTypeNumberRegister);
1280 }
1281
1282 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfNotInt(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1283 {
1284     move(reg1, scratch);
1285     and64(reg2, scratch);
1286     return emitJumpIfNotInt(scratch);
1287 }
1288
1289 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotInt(RegisterID reg)
1290 {
1291     addSlowCase(emitJumpIfNotInt(reg));
1292 }
1293
1294 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotInt(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1295 {
1296     addSlowCase(emitJumpIfNotInt(reg1, reg2, scratch));
1297 }
1298
1299 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotNumber(RegisterID reg)
1300 {
1301     addSlowCase(emitJumpIfNotNumber(reg));
1302 }
1303
1304 ALWAYS_INLINE void JIT::emitTagBool(RegisterID reg)
1305 {
1306     or32(TrustedImm32(static_cast<int32_t>(ValueFalse)), reg);
1307 }
1308
1309 #endif // USE(JSVALUE32_64)
1310
1311 } // namespace JSC
1312
1313 #endif // ENABLE(JIT)
1314
1315 #endif // JITInlines_h
1316