<https://bugs.webkit.org/show_bug.cgi?id=23049> [jsfunfuzz] With blocks do not correc...
[WebKit-https.git] / JavaScriptCore / jit / JIT.cpp
1 /*
2  * Copyright (C) 2008 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "JIT.h"
28
29 #if ENABLE(JIT)
30
31 #include "CodeBlock.h"
32 #include "JITInlineMethods.h"
33 #include "JSArray.h"
34 #include "JSFunction.h"
35 #include "Interpreter.h"
36 #include "ResultType.h"
37 #include "SamplingTool.h"
38
39 #ifndef NDEBUG
40 #include <stdio.h>
41 #endif
42
43 using namespace std;
44
45 namespace JSC {
46
47 COMPILE_ASSERT(STUB_ARGS_code == 0xC, STUB_ARGS_code_is_C);
48 COMPILE_ASSERT(STUB_ARGS_callFrame == 0xE, STUB_ARGS_callFrame_is_E);
49
50 #if COMPILER(GCC) && PLATFORM(X86)
51
52 #if PLATFORM(DARWIN)
53 #define SYMBOL_STRING(name) "_" #name
54 #else
55 #define SYMBOL_STRING(name) #name
56 #endif
57
58 asm(
59 ".globl " SYMBOL_STRING(ctiTrampoline) "\n"
60 SYMBOL_STRING(ctiTrampoline) ":" "\n"
61     "pushl %ebp" "\n"
62     "movl %esp, %ebp" "\n"
63     "pushl %esi" "\n"
64     "pushl %edi" "\n"
65     "pushl %ebx" "\n"
66     "subl $0x1c, %esp" "\n"
67     "movl $512, %esi" "\n"
68     "movl 0x38(%esp), %edi" "\n" // Ox38 = 0x0E * 4, 0x0E = STUB_ARGS_callFrame (see assertion above)
69     "call *0x30(%esp)" "\n" // Ox30 = 0x0C * 4, 0x0C = STUB_ARGS_code (see assertion above)
70     "addl $0x1c, %esp" "\n"
71     "popl %ebx" "\n"
72     "popl %edi" "\n"
73     "popl %esi" "\n"
74     "popl %ebp" "\n"
75     "ret" "\n"
76 );
77
78 asm(
79 ".globl " SYMBOL_STRING(ctiVMThrowTrampoline) "\n"
80 SYMBOL_STRING(ctiVMThrowTrampoline) ":" "\n"
81 #if USE(JIT_STUB_ARGUMENT_VA_LIST)
82     "call " SYMBOL_STRING(_ZN3JSC11Interpreter12cti_vm_throwEPvz) "\n"
83 #else
84 #if USE(JIT_STUB_ARGUMENT_REGISTER)
85     "movl %esp, %ecx" "\n"
86 #else // JIT_STUB_ARGUMENT_STACK
87     "movl %esp, 0(%esp)" "\n"
88 #endif
89     "call " SYMBOL_STRING(_ZN3JSC11Interpreter12cti_vm_throwEPPv) "\n"
90 #endif
91     "addl $0x1c, %esp" "\n"
92     "popl %ebx" "\n"
93     "popl %edi" "\n"
94     "popl %esi" "\n"
95     "popl %ebp" "\n"
96     "ret" "\n"
97 );
98     
99 #elif COMPILER(GCC) && PLATFORM(X86_64)
100
101 #if PLATFORM(DARWIN)
102 #define SYMBOL_STRING(name) "_" #name
103 #else
104 #define SYMBOL_STRING(name) #name
105 #endif
106
107 asm(
108 ".globl " SYMBOL_STRING(ctiTrampoline) "\n"
109 SYMBOL_STRING(ctiTrampoline) ":" "\n"
110     "pushq %rbp" "\n"
111     "movq %rsp, %rbp" "\n"
112     "pushq %r12" "\n"
113     "pushq %r13" "\n"
114     "pushq %rbx" "\n"
115     "subq $0x38, %rsp" "\n"
116     "movq $512, %r12" "\n"
117     "movq 0x70(%rsp), %r13" "\n" // Ox70 = 0x0E * 8, 0x0E = STUB_ARGS_callFrame (see assertion above)
118     "call *0x60(%rsp)" "\n" // Ox60 = 0x0C * 8, 0x0C = STUB_ARGS_code (see assertion above)
119     "addq $0x38, %rsp" "\n"
120     "popq %rbx" "\n"
121     "popq %r13" "\n"
122     "popq %r12" "\n"
123     "popq %rbp" "\n"
124     "ret" "\n"
125 );
126
127 asm(
128 ".globl " SYMBOL_STRING(ctiVMThrowTrampoline) "\n"
129 SYMBOL_STRING(ctiVMThrowTrampoline) ":" "\n"
130 #if USE(JIT_STUB_ARGUMENT_REGISTER)
131     "movq %rsp, %rdi" "\n"
132     "call " SYMBOL_STRING(_ZN3JSC11Interpreter12cti_vm_throwEPPv) "\n"
133 #else // JIT_STUB_ARGUMENT_VA_LIST or JIT_STUB_ARGUMENT_STACK
134 #error "JIT_STUB_ARGUMENT configuration not supported."
135 #endif
136     "addq $0x38, %rsp" "\n"
137     "popq %rbx" "\n"
138     "popq %r13" "\n"
139     "popq %r12" "\n"
140     "popq %rbp" "\n"
141     "ret" "\n"
142 );
143     
144 #elif COMPILER(MSVC)
145
146 extern "C" {
147     
148     __declspec(naked) JSValue* ctiTrampoline(void* code, RegisterFile*, CallFrame*, JSValue** exception, Profiler**, JSGlobalData*)
149     {
150         __asm {
151             push ebp;
152             mov ebp, esp;
153             push esi;
154             push edi;
155             push ebx;
156             sub esp, 0x1c;
157             mov esi, 512;
158             mov ecx, esp;
159             mov edi, [esp + 0x38];
160             call [esp + 0x30]; // Ox30 = 0x0C * 4, 0x0C = STUB_ARGS_code (see assertion above)
161             add esp, 0x1c;
162             pop ebx;
163             pop edi;
164             pop esi;
165             pop ebp;
166             ret;
167         }
168     }
169     
170     __declspec(naked) void ctiVMThrowTrampoline()
171     {
172         __asm {
173 #if USE(JIT_STUB_ARGUMENT_REGISTER)
174             mov ecx, esp;
175 #else // JIT_STUB_ARGUMENT_VA_LIST or JIT_STUB_ARGUMENT_STACK
176 #error "JIT_STUB_ARGUMENT configuration not supported."
177 #endif
178             call JSC::Interpreter::cti_vm_throw;
179             add esp, 0x1c;
180             pop ebx;
181             pop edi;
182             pop esi;
183             pop ebp;
184             ret;
185         }
186     }
187     
188 }
189
190 #endif
191
192 void ctiSetReturnAddress(void** where, void* what)
193 {
194     *where = what;
195 }
196
197 void ctiPatchCallByReturnAddress(void* where, void* what)
198 {
199     MacroAssembler::Jump::patch(where, what);
200 }
201
202 JIT::JIT(JSGlobalData* globalData, CodeBlock* codeBlock)
203     : m_interpreter(globalData->interpreter)
204     , m_globalData(globalData)
205     , m_codeBlock(codeBlock)
206     , m_labels(codeBlock ? codeBlock->instructions().size() : 0)
207     , m_propertyAccessCompilationInfo(codeBlock ? codeBlock->numberOfStructureStubInfos() : 0)
208     , m_callStructureStubCompilationInfo(codeBlock ? codeBlock->numberOfCallLinkInfos() : 0)
209     , m_lastResultBytecodeRegister(std::numeric_limits<int>::max())
210     , m_jumpTargetsPosition(0)
211 {
212 }
213
214 void JIT::compileOpStrictEq(Instruction* currentInstruction, CompileOpStrictEqType type)
215 {
216     bool negated = (type == OpNStrictEq);
217
218     unsigned dst = currentInstruction[1].u.operand;
219     unsigned src1 = currentInstruction[2].u.operand;
220     unsigned src2 = currentInstruction[3].u.operand;
221
222     emitGetVirtualRegisters(src1, X86::eax, src2, X86::edx);
223
224     // Check that bot are immediates, if so check if they're equal
225     Jump firstNotImmediate = emitJumpIfJSCell(X86::eax);
226     Jump secondNotImmediate = emitJumpIfJSCell(X86::edx);
227     Jump bothWereImmediatesButNotEqual = jne32(X86::edx, X86::eax);
228
229     // They are equal - set the result to true. (Or false, if negated).
230     move(Imm32(asInteger(jsBoolean(!negated))), X86::eax);
231     Jump bothWereImmediatesAndEqual = jump();
232
233     // eax was not an immediate, we haven't yet checked edx.
234     // If edx is also a JSCell, or is 0, then jump to a slow case,
235     // otherwise these values are not equal.
236     firstNotImmediate.link(this);
237     emitJumpSlowCaseIfJSCell(X86::edx);
238     addSlowCase(je32(X86::edx, Imm32(asInteger(JSImmediate::zeroImmediate()))));
239     Jump firstWasNotImmediate = jump();
240
241     // eax was an immediate, but edx wasn't.
242     // If eax is 0 jump to a slow case, otherwise these values are not equal.
243     secondNotImmediate.link(this);
244     addSlowCase(je32(X86::eax, Imm32(asInteger(JSImmediate::zeroImmediate()))));
245
246     // We get here if the two values are different immediates, or one is 0 and the other is a JSCell.
247     // Vaelues are not equal, set the result to false.
248     bothWereImmediatesButNotEqual.link(this);
249     firstWasNotImmediate.link(this);
250     move(Imm32(asInteger(jsBoolean(negated))), X86::eax);
251     
252     bothWereImmediatesAndEqual.link(this);
253     emitPutVirtualRegister(dst);
254 }
255
256 void JIT::emitSlowScriptCheck()
257 {
258     Jump skipTimeout = jnzSub32(Imm32(1), timeoutCheckRegister);
259     emitCTICall(Interpreter::cti_timeout_check);
260     move(X86::eax, timeoutCheckRegister);
261     skipTimeout.link(this);
262
263     killLastResultRegister();
264 }
265
266
267 #define NEXT_OPCODE(name) \
268     m_bytecodeIndex += OPCODE_LENGTH(name); \
269     break;
270
271 #define CTI_COMPILE_BINARY_OP(name) \
272     case name: { \
273         emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx); \
274         emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 2, X86::ecx); \
275         emitCTICall(Interpreter::cti_##name); \
276         emitPutVirtualRegister(currentInstruction[1].u.operand); \
277         NEXT_OPCODE(name); \
278     }
279
280 #define CTI_COMPILE_UNARY_OP(name) \
281     case name: { \
282         emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx); \
283         emitCTICall(Interpreter::cti_##name); \
284         emitPutVirtualRegister(currentInstruction[1].u.operand); \
285         NEXT_OPCODE(name); \
286     }
287
288 void JIT::privateCompileMainPass()
289 {
290     Instruction* instructionsBegin = m_codeBlock->instructions().begin();
291     unsigned instructionCount = m_codeBlock->instructions().size();
292     unsigned propertyAccessInstructionIndex = 0;
293     unsigned globalResolveInfoIndex = 0;
294     unsigned callLinkInfoIndex = 0;
295
296     for (m_bytecodeIndex = 0; m_bytecodeIndex < instructionCount; ) {
297         Instruction* currentInstruction = instructionsBegin + m_bytecodeIndex;
298         ASSERT_WITH_MESSAGE(m_interpreter->isOpcode(currentInstruction->u.opcode), "privateCompileMainPass gone bad @ %d", m_bytecodeIndex);
299
300 #if ENABLE(OPCODE_SAMPLING)
301         if (m_bytecodeIndex > 0) // Avoid the overhead of sampling op_enter twice.
302             store32(m_interpreter->sampler()->encodeSample(currentInstruction), m_interpreter->sampler()->sampleSlot());
303 #endif
304
305         m_labels[m_bytecodeIndex] = label();
306         OpcodeID opcodeID = m_interpreter->getOpcodeID(currentInstruction->u.opcode);
307
308         switch (opcodeID) {
309         case op_mov: {
310             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::eax);
311             emitPutVirtualRegister(currentInstruction[1].u.operand);
312             NEXT_OPCODE(op_mov);
313         }
314         case op_add: {
315             unsigned dst = currentInstruction[1].u.operand;
316             unsigned src1 = currentInstruction[2].u.operand;
317             unsigned src2 = currentInstruction[3].u.operand;
318
319             if (JSValue* value = getConstantImmediateNumericArg(src1)) {
320                 emitGetVirtualRegister(src2, X86::eax);
321                 emitJumpSlowCaseIfNotImmNum(X86::eax);
322                 addSlowCase(joAdd32(Imm32(getDeTaggedConstantImmediate(value)), X86::eax));
323                 signExtend32ToPtr(X86::eax, X86::eax);
324                 emitPutVirtualRegister(dst);
325             } else if (JSValue* value = getConstantImmediateNumericArg(src2)) {
326                 emitGetVirtualRegister(src1, X86::eax);
327                 emitJumpSlowCaseIfNotImmNum(X86::eax);
328                 addSlowCase(joAdd32(Imm32(getDeTaggedConstantImmediate(value)), X86::eax));
329                 signExtend32ToPtr(X86::eax, X86::eax);
330                 emitPutVirtualRegister(dst);
331             } else {
332                 OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
333                 if (types.first().mightBeNumber() && types.second().mightBeNumber())
334                     compileBinaryArithOp(op_add, currentInstruction[1].u.operand, currentInstruction[2].u.operand, currentInstruction[3].u.operand, OperandTypes::fromInt(currentInstruction[4].u.operand));
335                 else {
336                     emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
337                     emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 2, X86::ecx);
338                     emitCTICall(Interpreter::cti_op_add);
339                     emitPutVirtualRegister(currentInstruction[1].u.operand);
340                 }
341             }
342             NEXT_OPCODE(op_add);
343         }
344         case op_end: {
345             if (m_codeBlock->needsFullScopeChain())
346                 emitCTICall(Interpreter::cti_op_end);
347             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
348             push(Address(callFrameRegister, RegisterFile::ReturnPC * static_cast<int>(sizeof(Register))));
349             ret();
350             NEXT_OPCODE(op_end);
351         }
352         case op_jmp: {
353             unsigned target = currentInstruction[1].u.operand;
354             addJump(jump(), target + 1);
355             NEXT_OPCODE(op_jmp);
356         }
357         case op_pre_inc: {
358             int srcDst = currentInstruction[1].u.operand;
359             emitGetVirtualRegister(srcDst, X86::eax);
360             emitJumpSlowCaseIfNotImmNum(X86::eax);
361             addSlowCase(joAdd32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::eax));
362             signExtend32ToPtr(X86::eax, X86::eax);
363             emitPutVirtualRegister(srcDst);
364             NEXT_OPCODE(op_pre_inc);
365         }
366         case op_loop: {
367             emitSlowScriptCheck();
368
369             unsigned target = currentInstruction[1].u.operand;
370             addJump(jump(), target + 1);
371             NEXT_OPCODE(op_end);
372         }
373         case op_loop_if_less: {
374             emitSlowScriptCheck();
375
376             unsigned target = currentInstruction[3].u.operand;
377             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
378             if (src2imm) {
379                 emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
380                 emitJumpSlowCaseIfNotImmNum(X86::eax);
381                 addJump(jl32(X86::eax, Imm32(asInteger(src2imm))), target + 3);
382             } else {
383                 emitGetVirtualRegisters(currentInstruction[1].u.operand, X86::eax, currentInstruction[2].u.operand, X86::edx);
384                 emitJumpSlowCaseIfNotImmNum(X86::eax);
385                 emitJumpSlowCaseIfNotImmNum(X86::edx);
386                 addJump(jl32(X86::eax, X86::edx), target + 3);
387             }
388             NEXT_OPCODE(op_loop_if_less);
389         }
390         case op_loop_if_lesseq: {
391             emitSlowScriptCheck();
392
393             unsigned target = currentInstruction[3].u.operand;
394             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
395             if (src2imm) {
396                 emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
397                 emitJumpSlowCaseIfNotImmNum(X86::eax);
398                 addJump(jle32(X86::eax, Imm32(asInteger(src2imm))), target + 3);
399             } else {
400                 emitGetVirtualRegisters(currentInstruction[1].u.operand, X86::eax, currentInstruction[2].u.operand, X86::edx);
401                 emitJumpSlowCaseIfNotImmNum(X86::eax);
402                 emitJumpSlowCaseIfNotImmNum(X86::edx);
403                 addJump(jle32(X86::eax, X86::edx), target + 3);
404             }
405             NEXT_OPCODE(op_loop_if_lesseq);
406         }
407         case op_new_object: {
408             emitCTICall(Interpreter::cti_op_new_object);
409             emitPutVirtualRegister(currentInstruction[1].u.operand);
410             NEXT_OPCODE(op_new_object);
411         }
412         case op_put_by_id: {
413             compilePutByIdHotPath(currentInstruction[1].u.operand, &(m_codeBlock->identifier(currentInstruction[2].u.operand)), currentInstruction[3].u.operand, propertyAccessInstructionIndex++);
414             NEXT_OPCODE(op_put_by_id);
415         }
416         case op_get_by_id: {
417             compileGetByIdHotPath(currentInstruction[1].u.operand, currentInstruction[2].u.operand, &(m_codeBlock->identifier(currentInstruction[3].u.operand)), propertyAccessInstructionIndex++);
418             NEXT_OPCODE(op_get_by_id);
419         }
420         case op_instanceof: {
421             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::eax); // value
422             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::ecx); // baseVal
423             emitGetVirtualRegister(currentInstruction[4].u.operand, X86::edx); // proto
424
425             // check if any are immediates
426             move(X86::eax, X86::ebx);
427             or32(X86::ecx, X86::ebx);
428             or32(X86::edx, X86::ebx);
429             emitJumpSlowCaseIfNotJSCell(X86::ebx);
430
431             // check that all are object type - this is a bit of a bithack to avoid excess branching;
432             // we check that the sum of the three type codes from Structures is exactly 3 * ObjectType,
433             // this works because NumberType and StringType are smaller
434             move(Imm32(3 * ObjectType), X86::ebx);
435             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::eax);
436             loadPtr(Address(X86::ecx, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
437             loadPtr(Address(X86::edx, FIELD_OFFSET(JSCell, m_structure)), X86::edx);
438             sub32(Address(X86::eax, FIELD_OFFSET(Structure, m_typeInfo.m_type)), X86::ebx);
439             sub32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_type)), X86::ebx);
440             addSlowCase(jne32(Address(X86::edx, FIELD_OFFSET(Structure, m_typeInfo.m_type)), X86::ebx));
441
442             // check that baseVal's flags include ImplementsHasInstance but not OverridesHasInstance
443             load32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), X86::ecx);
444             and32(Imm32(ImplementsHasInstance | OverridesHasInstance), X86::ecx);
445             addSlowCase(jne32(X86::ecx, Imm32(ImplementsHasInstance)));
446
447             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::ecx); // reload value
448             emitGetVirtualRegister(currentInstruction[4].u.operand, X86::edx); // reload proto
449
450             // optimistically load true result
451             move(Imm32(asInteger(jsBoolean(true))), X86::eax);
452
453             Label loop(this);
454
455             // load value's prototype
456             loadPtr(Address(X86::ecx, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
457             loadPtr(Address(X86::ecx, FIELD_OFFSET(Structure, m_prototype)), X86::ecx);
458
459             Jump exit = jePtr(X86::ecx, X86::edx);
460
461             jne32(X86::ecx, Imm32(asInteger(jsNull())), loop);
462
463             move(Imm32(asInteger(jsBoolean(false))), X86::eax);
464
465             exit.link(this);
466
467             emitPutVirtualRegister(currentInstruction[1].u.operand);
468
469             NEXT_OPCODE(op_instanceof);
470         }
471         case op_del_by_id: {
472             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
473             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[3].u.operand));
474             emitPutJITStubArgConstant(ident, 2);
475             emitCTICall(Interpreter::cti_op_del_by_id);
476             emitPutVirtualRegister(currentInstruction[1].u.operand);
477             NEXT_OPCODE(op_del_by_id);
478         }
479         case op_mul: {
480             unsigned dst = currentInstruction[1].u.operand;
481             unsigned src1 = currentInstruction[2].u.operand;
482             unsigned src2 = currentInstruction[3].u.operand;
483
484             // For now, only plant a fast int case if the constant operand is greater than zero.
485             JSValue* src1Value = getConstantImmediateNumericArg(src1);
486             JSValue* src2Value = getConstantImmediateNumericArg(src2);
487             int32_t value;
488             if (src1Value && ((value = JSImmediate::intValue(src1Value)) > 0)) {
489                 emitGetVirtualRegister(src2, X86::eax);
490                 emitJumpSlowCaseIfNotImmNum(X86::eax);
491                 emitFastArithDeTagImmediate(X86::eax);
492                 addSlowCase(joMul32(Imm32(value), X86::eax, X86::eax));
493                 emitFastArithReTagImmediate(X86::eax);
494                 emitPutVirtualRegister(dst);
495             } else if (src2Value && ((value = JSImmediate::intValue(src2Value)) > 0)) {
496                 emitGetVirtualRegister(src1, X86::eax);
497                 emitJumpSlowCaseIfNotImmNum(X86::eax);
498                 emitFastArithDeTagImmediate(X86::eax);
499                 addSlowCase(joMul32(Imm32(value), X86::eax, X86::eax));
500                 emitFastArithReTagImmediate(X86::eax);
501                 emitPutVirtualRegister(dst);
502             } else
503                 compileBinaryArithOp(op_mul, currentInstruction[1].u.operand, currentInstruction[2].u.operand, currentInstruction[3].u.operand, OperandTypes::fromInt(currentInstruction[4].u.operand));
504
505             NEXT_OPCODE(op_mul);
506         }
507         case op_new_func: {
508             FuncDeclNode* func = m_codeBlock->function(currentInstruction[2].u.operand);
509             emitPutJITStubArgConstant(func, 1);
510             emitCTICall(Interpreter::cti_op_new_func);
511             emitPutVirtualRegister(currentInstruction[1].u.operand);
512             NEXT_OPCODE(op_new_func);
513         }
514         case op_call: {
515             compileOpCall(opcodeID, currentInstruction, callLinkInfoIndex++);
516             NEXT_OPCODE(op_call);
517         }
518         case op_call_eval: {
519             compileOpCall(opcodeID, currentInstruction, callLinkInfoIndex++);
520             NEXT_OPCODE(op_call_eval);
521         }
522         case op_construct: {
523             compileOpCall(opcodeID, currentInstruction, callLinkInfoIndex++);
524             NEXT_OPCODE(op_construct);
525         }
526         case op_get_global_var: {
527             JSVariableObject* globalObject = static_cast<JSVariableObject*>(currentInstruction[2].u.jsCell);
528             move(ImmPtr(globalObject), X86::eax);
529             emitGetVariableObjectRegister(X86::eax, currentInstruction[3].u.operand, X86::eax);
530             emitPutVirtualRegister(currentInstruction[1].u.operand);
531             NEXT_OPCODE(op_get_global_var);
532         }
533         case op_put_global_var: {
534             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::edx);
535             JSVariableObject* globalObject = static_cast<JSVariableObject*>(currentInstruction[1].u.jsCell);
536             move(ImmPtr(globalObject), X86::eax);
537             emitPutVariableObjectRegister(X86::edx, X86::eax, currentInstruction[2].u.operand);
538             NEXT_OPCODE(op_put_global_var);
539         }
540         case op_get_scoped_var: {
541             int skip = currentInstruction[3].u.operand + m_codeBlock->needsFullScopeChain();
542
543             emitGetFromCallFrameHeader(RegisterFile::ScopeChain, X86::eax);
544             while (skip--)
545                 loadPtr(Address(X86::eax, FIELD_OFFSET(ScopeChainNode, next)), X86::eax);
546
547             loadPtr(Address(X86::eax, FIELD_OFFSET(ScopeChainNode, object)), X86::eax);
548             emitGetVariableObjectRegister(X86::eax, currentInstruction[2].u.operand, X86::eax);
549             emitPutVirtualRegister(currentInstruction[1].u.operand);
550             NEXT_OPCODE(op_get_scoped_var);
551         }
552         case op_put_scoped_var: {
553             int skip = currentInstruction[2].u.operand + m_codeBlock->needsFullScopeChain();
554
555             emitGetFromCallFrameHeader(RegisterFile::ScopeChain, X86::edx);
556             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::eax);
557             while (skip--)
558                 loadPtr(Address(X86::edx, FIELD_OFFSET(ScopeChainNode, next)), X86::edx);
559
560             loadPtr(Address(X86::edx, FIELD_OFFSET(ScopeChainNode, object)), X86::edx);
561             emitPutVariableObjectRegister(X86::eax, X86::edx, currentInstruction[1].u.operand);
562             NEXT_OPCODE(op_put_scoped_var);
563         }
564         case op_tear_off_activation: {
565             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
566             emitCTICall(Interpreter::cti_op_tear_off_activation);
567             NEXT_OPCODE(op_tear_off_activation);
568         }
569         case op_tear_off_arguments: {
570             emitCTICall(Interpreter::cti_op_tear_off_arguments);
571             NEXT_OPCODE(op_tear_off_arguments);
572         }
573         case op_ret: {
574             // We could JIT generate the deref, only calling out to C when the refcount hits zero.
575             if (m_codeBlock->needsFullScopeChain())
576                 emitCTICall(Interpreter::cti_op_ret_scopeChain);
577
578             // Return the result in %eax.
579             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
580
581             // Grab the return address.
582             emitGetFromCallFrameHeader(RegisterFile::ReturnPC, X86::edx);
583
584             // Restore our caller's "r".
585             emitGetFromCallFrameHeader(RegisterFile::CallerFrame, callFrameRegister);
586
587             // Return.
588             push(X86::edx);
589             ret();
590
591             NEXT_OPCODE(op_ret);
592         }
593         case op_new_array: {
594             emitPutJITStubArgConstant(currentInstruction[2].u.operand, 1);
595             emitPutJITStubArgConstant(currentInstruction[3].u.operand, 2);
596             emitCTICall(Interpreter::cti_op_new_array);
597             emitPutVirtualRegister(currentInstruction[1].u.operand);
598             NEXT_OPCODE(op_new_array);
599         }
600         case op_resolve: {
601             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
602             emitPutJITStubArgConstant(ident, 1);
603             emitCTICall(Interpreter::cti_op_resolve);
604             emitPutVirtualRegister(currentInstruction[1].u.operand);
605             NEXT_OPCODE(op_resolve);
606         }
607         case op_construct_verify: {
608             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
609
610             emitJumpSlowCaseIfNotJSCell(X86::eax);
611             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
612             addSlowCase(jne32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo) + FIELD_OFFSET(TypeInfo, m_type)), Imm32(ObjectType)));
613
614             NEXT_OPCODE(op_construct_verify);
615         }
616         case op_get_by_val: {
617             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::edx);
618             emitJumpSlowCaseIfNotImmNum(X86::edx);
619             emitFastArithImmToInt(X86::edx);
620             emitJumpSlowCaseIfNotJSCell(X86::eax);
621             addSlowCase(jnePtr(Address(X86::eax), ImmPtr(m_interpreter->m_jsArrayVptr)));
622
623             // This is an array; get the m_storage pointer into ecx, then check if the index is below the fast cutoff
624             loadPtr(Address(X86::eax, FIELD_OFFSET(JSArray, m_storage)), X86::ecx);
625             addSlowCase(jae32(X86::edx, Address(X86::eax, FIELD_OFFSET(JSArray, m_fastAccessCutoff))));
626
627             // Get the value from the vector
628             loadPtr(BaseIndex(X86::ecx, X86::edx, ScalePtr, FIELD_OFFSET(ArrayStorage, m_vector[0])), X86::eax);
629             emitPutVirtualRegister(currentInstruction[1].u.operand);
630             NEXT_OPCODE(op_get_by_val);
631         }
632         case op_resolve_func: {
633             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[3].u.operand));
634             emitPutJITStubArgConstant(ident, 1);
635             emitCTICall(Interpreter::cti_op_resolve_func);
636             emitPutVirtualRegister(currentInstruction[2].u.operand, X86::edx);
637             emitPutVirtualRegister(currentInstruction[1].u.operand);
638             NEXT_OPCODE(op_resolve_func);
639         }
640         case op_sub: {
641             compileBinaryArithOp(op_sub, currentInstruction[1].u.operand, currentInstruction[2].u.operand, currentInstruction[3].u.operand, OperandTypes::fromInt(currentInstruction[4].u.operand));
642             NEXT_OPCODE(op_sub);
643         }
644         case op_put_by_val: {
645             emitGetVirtualRegisters(currentInstruction[1].u.operand, X86::eax, currentInstruction[2].u.operand, X86::edx);
646             emitJumpSlowCaseIfNotImmNum(X86::edx);
647             emitFastArithImmToInt(X86::edx);
648             emitJumpSlowCaseIfNotJSCell(X86::eax);
649             addSlowCase(jnePtr(Address(X86::eax), ImmPtr(m_interpreter->m_jsArrayVptr)));
650
651             // This is an array; get the m_storage pointer into ecx, then check if the index is below the fast cutoff
652             loadPtr(Address(X86::eax, FIELD_OFFSET(JSArray, m_storage)), X86::ecx);
653             Jump inFastVector = jb32(X86::edx, Address(X86::eax, FIELD_OFFSET(JSArray, m_fastAccessCutoff)));
654             // No; oh well, check if the access if within the vector - if so, we may still be okay.
655             addSlowCase(jae32(X86::edx, Address(X86::ecx, FIELD_OFFSET(ArrayStorage, m_vectorLength))));
656
657             // This is a write to the slow part of the vector; first, we have to check if this would be the first write to this location.
658             // FIXME: should be able to handle initial write to array; increment the the number of items in the array, and potentially update fast access cutoff. 
659             addSlowCase(jzPtr(BaseIndex(X86::ecx, X86::edx, ScalePtr, FIELD_OFFSET(ArrayStorage, m_vector[0]))));
660
661             // All good - put the value into the array.
662             inFastVector.link(this);
663             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::eax);
664             storePtr(X86::eax, BaseIndex(X86::ecx, X86::edx, ScalePtr, FIELD_OFFSET(ArrayStorage, m_vector[0])));
665             NEXT_OPCODE(op_put_by_val);
666         }
667         CTI_COMPILE_BINARY_OP(op_lesseq)
668         case op_loop_if_true: {
669             emitSlowScriptCheck();
670
671             unsigned target = currentInstruction[2].u.operand;
672             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
673
674             Jump isZero = je32(X86::eax, Imm32(asInteger(JSImmediate::zeroImmediate())));
675             addJump(jnz32(X86::eax, Imm32(JSImmediate::TagBitTypeInteger)), target + 2);
676
677             addJump(je32(X86::eax, Imm32(asInteger(JSImmediate::trueImmediate()))), target + 2);
678             addSlowCase(jne32(X86::eax, Imm32(asInteger(JSImmediate::falseImmediate()))));
679
680             isZero.link(this);
681             NEXT_OPCODE(op_loop_if_true);
682         };
683         case op_resolve_base: {
684             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
685             emitPutJITStubArgConstant(ident, 1);
686             emitCTICall(Interpreter::cti_op_resolve_base);
687             emitPutVirtualRegister(currentInstruction[1].u.operand);
688             NEXT_OPCODE(op_resolve_base);
689         }
690         case op_negate: {
691             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
692             emitCTICall(Interpreter::cti_op_negate);
693             emitPutVirtualRegister(currentInstruction[1].u.operand);
694             NEXT_OPCODE(op_negate);
695         }
696         case op_resolve_skip: {
697             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
698             emitPutJITStubArgConstant(ident, 1);
699             emitPutJITStubArgConstant(currentInstruction[3].u.operand + m_codeBlock->needsFullScopeChain(), 2);
700             emitCTICall(Interpreter::cti_op_resolve_skip);
701             emitPutVirtualRegister(currentInstruction[1].u.operand);
702             NEXT_OPCODE(op_resolve_skip);
703         }
704         case op_resolve_global: {
705             // Fast case
706             void* globalObject = currentInstruction[2].u.jsCell;
707             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[3].u.operand));
708             
709             unsigned currentIndex = globalResolveInfoIndex++;
710             void* structureAddress = &(m_codeBlock->globalResolveInfo(currentIndex).structure);
711             void* offsetAddr = &(m_codeBlock->globalResolveInfo(currentIndex).offset);
712
713             // Check Structure of global object
714             move(ImmPtr(globalObject), X86::eax);
715             loadPtr(structureAddress, X86::edx);
716             Jump noMatch = jnePtr(X86::edx, Address(X86::eax, FIELD_OFFSET(JSCell, m_structure))); // Structures don't match
717
718             // Load cached property
719             loadPtr(Address(X86::eax, FIELD_OFFSET(JSGlobalObject, m_propertyStorage)), X86::eax);
720             load32(offsetAddr, X86::edx);
721             loadPtr(BaseIndex(X86::eax, X86::edx, ScalePtr), X86::eax);
722             emitPutVirtualRegister(currentInstruction[1].u.operand);
723             Jump end = jump();
724
725             // Slow case
726             noMatch.link(this);
727             emitPutJITStubArgConstant(globalObject, 1);
728             emitPutJITStubArgConstant(ident, 2);
729             emitPutJITStubArgConstant(currentIndex, 3);
730             emitCTICall(Interpreter::cti_op_resolve_global);
731             emitPutVirtualRegister(currentInstruction[1].u.operand);
732             end.link(this);
733             NEXT_OPCODE(op_resolve_global);
734         }
735         CTI_COMPILE_BINARY_OP(op_div)
736         case op_pre_dec: {
737             int srcDst = currentInstruction[1].u.operand;
738             emitGetVirtualRegister(srcDst, X86::eax);
739             emitJumpSlowCaseIfNotImmNum(X86::eax);
740             addSlowCase(joSub32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::eax));
741             signExtend32ToPtr(X86::eax, X86::eax);
742             emitPutVirtualRegister(srcDst);
743             NEXT_OPCODE(op_pre_dec);
744         }
745         case op_jnless: {
746             unsigned target = currentInstruction[3].u.operand;
747             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
748             if (src2imm) {
749                 emitGetVirtualRegister(currentInstruction[1].u.operand, X86::edx);
750                 emitJumpSlowCaseIfNotImmNum(X86::edx);
751                 addJump(jge32(X86::edx, Imm32(asInteger(src2imm))), target + 3);
752             } else {
753                 emitGetVirtualRegisters(currentInstruction[1].u.operand, X86::eax, currentInstruction[2].u.operand, X86::edx);
754                 emitJumpSlowCaseIfNotImmNum(X86::eax);
755                 emitJumpSlowCaseIfNotImmNum(X86::edx);
756                 addJump(jge32(X86::eax, X86::edx), target + 3);
757             }
758             NEXT_OPCODE(op_jnless);
759         }
760         case op_not: {
761             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::eax);
762             xorPtr(Imm32(JSImmediate::FullTagTypeBool), X86::eax);
763             addSlowCase(jnz32(X86::eax, Imm32(JSImmediate::FullTagTypeMask)));
764             xorPtr(Imm32(JSImmediate::FullTagTypeBool | JSImmediate::ExtendedPayloadBitBoolValue), X86::eax);
765             emitPutVirtualRegister(currentInstruction[1].u.operand);
766             NEXT_OPCODE(op_not);
767         }
768         case op_jfalse: {
769             unsigned target = currentInstruction[2].u.operand;
770             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
771
772             addJump(je32(X86::eax, Imm32(asInteger(JSImmediate::zeroImmediate()))), target + 2);
773             Jump isNonZero = jnz32(X86::eax, Imm32(JSImmediate::TagBitTypeInteger));
774
775             addJump(je32(X86::eax, Imm32(asInteger(JSImmediate::falseImmediate()))), target + 2);
776             addSlowCase(jne32(X86::eax, Imm32(asInteger(JSImmediate::trueImmediate()))));
777
778             isNonZero.link(this);
779             NEXT_OPCODE(op_jfalse);
780         };
781         case op_jeq_null: {
782             unsigned src = currentInstruction[1].u.operand;
783             unsigned target = currentInstruction[2].u.operand;
784
785             emitGetVirtualRegister(src, X86::eax);
786             Jump isImmediate = emitJumpIfNotJSCell(X86::eax);
787
788             // First, handle JSCell cases - check MasqueradesAsUndefined bit on the structure.
789             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
790             addJump(jnz32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), Imm32(MasqueradesAsUndefined)), target + 2);
791             Jump wasNotImmediate = jump();
792
793             // Now handle the immediate cases - undefined & null
794             isImmediate.link(this);
795             and32(Imm32(~JSImmediate::ExtendedTagBitUndefined), X86::eax);
796             addJump(je32(X86::eax, Imm32(asInteger(jsNull()))), target + 2);            
797
798             wasNotImmediate.link(this);
799             NEXT_OPCODE(op_jeq_null);
800         };
801         case op_jneq_null: {
802             unsigned src = currentInstruction[1].u.operand;
803             unsigned target = currentInstruction[2].u.operand;
804
805             emitGetVirtualRegister(src, X86::eax);
806             Jump isImmediate = emitJumpIfNotJSCell(X86::eax);
807
808             // First, handle JSCell cases - check MasqueradesAsUndefined bit on the structure.
809             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
810             addJump(jz32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), Imm32(MasqueradesAsUndefined)), target + 2);
811             Jump wasNotImmediate = jump();
812
813             // Now handle the immediate cases - undefined & null
814             isImmediate.link(this);
815             and32(Imm32(~JSImmediate::ExtendedTagBitUndefined), X86::eax);
816             addJump(jne32(X86::eax, Imm32(asInteger(jsNull()))), target + 2);            
817
818             wasNotImmediate.link(this);
819             NEXT_OPCODE(op_jneq_null);
820         }
821         case op_post_inc: {
822             int srcDst = currentInstruction[2].u.operand;
823             emitGetVirtualRegister(srcDst, X86::eax);
824             move(X86::eax, X86::edx);
825             emitJumpSlowCaseIfNotImmNum(X86::eax);
826             addSlowCase(joAdd32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::edx));
827             signExtend32ToPtr(X86::edx, X86::edx);
828             emitPutVirtualRegister(srcDst, X86::edx);
829             emitPutVirtualRegister(currentInstruction[1].u.operand);
830             NEXT_OPCODE(op_post_inc);
831         }
832         case op_unexpected_load: {
833             JSValue* v = m_codeBlock->unexpectedConstant(currentInstruction[2].u.operand);
834             move(ImmPtr(v), X86::eax);
835             emitPutVirtualRegister(currentInstruction[1].u.operand);
836             NEXT_OPCODE(op_unexpected_load);
837         }
838         case op_jsr: {
839             int retAddrDst = currentInstruction[1].u.operand;
840             int target = currentInstruction[2].u.operand;
841             DataLabelPtr storeLocation = storePtrWithPatch(Address(callFrameRegister, sizeof(Register) * retAddrDst));
842             addJump(jump(), target + 2);
843             m_jsrSites.append(JSRInfo(storeLocation, label()));
844             NEXT_OPCODE(op_jsr);
845         }
846         case op_sret: {
847             jump(Address(callFrameRegister, sizeof(Register) * currentInstruction[1].u.operand));
848             NEXT_OPCODE(op_sret);
849         }
850         case op_eq: {
851             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::edx);
852             emitJumpSlowCaseIfNotImmNums(X86::eax, X86::edx, X86::ecx);
853             sete32(X86::edx, X86::eax);
854             emitTagAsBoolImmediate(X86::eax);
855             emitPutVirtualRegister(currentInstruction[1].u.operand);
856             NEXT_OPCODE(op_eq);
857         }
858         case op_lshift: {
859             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::ecx);
860             emitJumpSlowCaseIfNotImmNum(X86::eax);
861             emitJumpSlowCaseIfNotImmNum(X86::ecx);
862             emitFastArithImmToInt(X86::eax);
863             emitFastArithImmToInt(X86::ecx);
864             lshift32(X86::ecx, X86::eax);
865             emitFastArithIntToImmOrSlowCase(X86::eax);
866             emitPutVirtualRegister(currentInstruction[1].u.operand);
867             NEXT_OPCODE(op_lshift);
868         }
869         case op_bitand: {
870             unsigned src1 = currentInstruction[2].u.operand;
871             unsigned src2 = currentInstruction[3].u.operand;
872             unsigned dst = currentInstruction[1].u.operand;
873             if (JSValue* value = getConstantImmediateNumericArg(src1)) {
874                 emitGetVirtualRegister(src2, X86::eax);
875                 emitJumpSlowCaseIfNotImmNum(X86::eax);
876                 andPtr(Imm32(asInteger(value)), X86::eax); // FIXME: make it more obvious this is relying on the format of JSImmediate
877                 emitPutVirtualRegister(dst);
878             } else if (JSValue* value = getConstantImmediateNumericArg(src2)) {
879                 emitGetVirtualRegister(src1, X86::eax);
880                 emitJumpSlowCaseIfNotImmNum(X86::eax);
881                 andPtr(Imm32(asInteger(value)), X86::eax);
882                 emitPutVirtualRegister(dst);
883             } else {
884                 emitGetVirtualRegisters(src1, X86::eax, src2, X86::edx);
885                 andPtr(X86::edx, X86::eax);
886                 emitJumpSlowCaseIfNotImmNum(X86::eax);
887                 emitPutVirtualRegister(dst);
888             }
889             NEXT_OPCODE(op_bitand);
890         }
891         case op_rshift: {
892             unsigned src1 = currentInstruction[2].u.operand;
893             unsigned src2 = currentInstruction[3].u.operand;
894             if (JSValue* value = getConstantImmediateNumericArg(src2)) {
895                 emitGetVirtualRegister(src1, X86::eax);
896                 emitJumpSlowCaseIfNotImmNum(X86::eax);
897                 // Mask with 0x1f as per ecma-262 11.7.2 step 7.
898                 rshift32(Imm32(JSImmediate::getTruncatedUInt32(value) & 0x1f), X86::eax);
899             } else {
900                 emitGetVirtualRegisters(src1, X86::eax, src2, X86::ecx);
901                 emitJumpSlowCaseIfNotImmNum(X86::eax);
902                 emitJumpSlowCaseIfNotImmNum(X86::ecx);
903                 emitFastArithImmToInt(X86::ecx);
904                 rshift32(X86::ecx, X86::eax);
905             }
906             emitFastArithPotentiallyReTagImmediate(X86::eax);
907             emitPutVirtualRegister(currentInstruction[1].u.operand);
908             NEXT_OPCODE(op_rshift);
909         }
910         case op_bitnot: {
911             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::eax);
912             emitJumpSlowCaseIfNotImmNum(X86::eax);
913             xorPtr(Imm32(~JSImmediate::TagBitTypeInteger), X86::eax);
914             emitPutVirtualRegister(currentInstruction[1].u.operand);
915             NEXT_OPCODE(op_bitnot);
916         }
917         case op_resolve_with_base: {
918             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[3].u.operand));
919             emitPutJITStubArgConstant(ident, 1);
920             emitCTICall(Interpreter::cti_op_resolve_with_base);
921             emitPutVirtualRegister(currentInstruction[2].u.operand, X86::edx);
922             emitPutVirtualRegister(currentInstruction[1].u.operand);
923             NEXT_OPCODE(op_resolve_with_base);
924         }
925         case op_new_func_exp: {
926             FuncExprNode* func = m_codeBlock->functionExpression(currentInstruction[2].u.operand);
927             emitPutJITStubArgConstant(func, 1);
928             emitCTICall(Interpreter::cti_op_new_func_exp);
929             emitPutVirtualRegister(currentInstruction[1].u.operand);
930             NEXT_OPCODE(op_new_func_exp);
931         }
932         case op_mod: {
933             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::ecx);
934             emitJumpSlowCaseIfNotImmNum(X86::eax);
935             emitJumpSlowCaseIfNotImmNum(X86::ecx);
936             emitFastArithDeTagImmediate(X86::eax);
937             addSlowCase(emitFastArithDeTagImmediateJumpIfZero(X86::ecx));
938             mod32(X86::ecx, X86::eax, X86::edx);
939             emitFastArithReTagImmediate(X86::edx);
940             move(X86::edx, X86::eax);
941             emitPutVirtualRegister(currentInstruction[1].u.operand);
942             NEXT_OPCODE(op_mod);
943         }
944         case op_jtrue: {
945             unsigned target = currentInstruction[2].u.operand;
946             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
947
948             Jump isZero = je32(X86::eax, Imm32(asInteger(JSImmediate::zeroImmediate())));
949             addJump(jnz32(X86::eax, Imm32(JSImmediate::TagBitTypeInteger)), target + 2);
950
951             addJump(je32(X86::eax, Imm32(asInteger(JSImmediate::trueImmediate()))), target + 2);
952             addSlowCase(jne32(X86::eax, Imm32(asInteger(JSImmediate::falseImmediate()))));
953
954             isZero.link(this);
955             NEXT_OPCODE(op_jtrue);
956         }
957         CTI_COMPILE_BINARY_OP(op_less)
958         case op_neq: {
959             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::edx);
960             emitJumpSlowCaseIfNotImmNums(X86::eax, X86::edx, X86::ecx);
961             setne32(X86::edx, X86::eax);
962             emitTagAsBoolImmediate(X86::eax);
963
964             emitPutVirtualRegister(currentInstruction[1].u.operand);
965
966             NEXT_OPCODE(op_neq);
967         }
968         case op_post_dec: {
969             int srcDst = currentInstruction[2].u.operand;
970             emitGetVirtualRegister(srcDst, X86::eax);
971             move(X86::eax, X86::edx);
972             emitJumpSlowCaseIfNotImmNum(X86::eax);
973             addSlowCase(joSub32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::edx));
974             signExtend32ToPtr(X86::edx, X86::edx);
975             emitPutVirtualRegister(srcDst, X86::edx);
976             emitPutVirtualRegister(currentInstruction[1].u.operand);
977             NEXT_OPCODE(op_post_dec);
978         }
979         CTI_COMPILE_BINARY_OP(op_urshift)
980         case op_bitxor: {
981             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::edx);
982             emitJumpSlowCaseIfNotImmNums(X86::eax, X86::edx, X86::ecx);
983             xor32(X86::edx, X86::eax);
984             emitFastArithReTagImmediate(X86::eax);
985             emitPutVirtualRegister(currentInstruction[1].u.operand);
986             NEXT_OPCODE(op_bitxor);
987         }
988         case op_new_regexp: {
989             RegExp* regExp = m_codeBlock->regexp(currentInstruction[2].u.operand);
990             emitPutJITStubArgConstant(regExp, 1);
991             emitCTICall(Interpreter::cti_op_new_regexp);
992             emitPutVirtualRegister(currentInstruction[1].u.operand);
993             NEXT_OPCODE(op_new_regexp);
994         }
995         case op_bitor: {
996             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::edx);
997             emitJumpSlowCaseIfNotImmNums(X86::eax, X86::edx, X86::ecx);
998             orPtr(X86::edx, X86::eax);
999             emitPutVirtualRegister(currentInstruction[1].u.operand);
1000             NEXT_OPCODE(op_bitor);
1001         }
1002         case op_throw: {
1003             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
1004             emitCTICall(Interpreter::cti_op_throw);
1005 #if PLATFORM(X86_64)
1006             addPtr(Imm32(0x38), X86::esp);
1007             pop(X86::ebx);
1008             pop(X86::r13);
1009             pop(X86::r12);
1010             pop(X86::ebp);
1011             ret();
1012 #else
1013             addPtr(Imm32(0x1c), X86::esp);
1014             pop(X86::ebx);
1015             pop(X86::edi);
1016             pop(X86::esi);
1017             pop(X86::ebp);
1018             ret();
1019 #endif
1020             NEXT_OPCODE(op_throw);
1021         }
1022         case op_get_pnames: {
1023             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
1024             emitCTICall(Interpreter::cti_op_get_pnames);
1025             emitPutVirtualRegister(currentInstruction[1].u.operand);
1026             NEXT_OPCODE(op_get_pnames);
1027         }
1028         case op_next_pname: {
1029             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
1030             unsigned target = currentInstruction[3].u.operand;
1031             emitCTICall(Interpreter::cti_op_next_pname);
1032             Jump endOfIter = jzPtr(X86::eax);
1033             emitPutVirtualRegister(currentInstruction[1].u.operand);
1034             addJump(jump(), target + 3);
1035             endOfIter.link(this);
1036             NEXT_OPCODE(op_next_pname);
1037         }
1038         case op_push_scope: {
1039             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
1040             emitCTICall(Interpreter::cti_op_push_scope);
1041             emitPutVirtualRegister(currentInstruction[1].u.operand);
1042             NEXT_OPCODE(op_push_scope);
1043         }
1044         case op_pop_scope: {
1045             emitCTICall(Interpreter::cti_op_pop_scope);
1046             NEXT_OPCODE(op_pop_scope);
1047         }
1048         CTI_COMPILE_UNARY_OP(op_typeof)
1049         CTI_COMPILE_UNARY_OP(op_is_undefined)
1050         CTI_COMPILE_UNARY_OP(op_is_boolean)
1051         CTI_COMPILE_UNARY_OP(op_is_number)
1052         CTI_COMPILE_UNARY_OP(op_is_string)
1053         CTI_COMPILE_UNARY_OP(op_is_object)
1054         CTI_COMPILE_UNARY_OP(op_is_function)
1055         case op_stricteq: {
1056             compileOpStrictEq(currentInstruction, OpStrictEq);
1057             NEXT_OPCODE(op_stricteq);
1058         }
1059         case op_nstricteq: {
1060             compileOpStrictEq(currentInstruction, OpNStrictEq);
1061             NEXT_OPCODE(op_nstricteq);
1062         }
1063         case op_to_jsnumber: {
1064             int srcVReg = currentInstruction[2].u.operand;
1065             emitGetVirtualRegister(srcVReg, X86::eax);
1066             
1067             Jump wasImmediate = jnz32(X86::eax, Imm32(JSImmediate::TagBitTypeInteger));
1068
1069             emitJumpSlowCaseIfNotJSCell(X86::eax, srcVReg);
1070             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
1071             addSlowCase(jne32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_type)), Imm32(NumberType)));
1072             
1073             wasImmediate.link(this);
1074
1075             emitPutVirtualRegister(currentInstruction[1].u.operand);
1076             NEXT_OPCODE(op_to_jsnumber);
1077         }
1078         CTI_COMPILE_BINARY_OP(op_in)
1079         case op_push_new_scope: {
1080             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
1081             emitPutJITStubArgConstant(ident, 1);
1082             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 2, X86::ecx);
1083             emitCTICall(Interpreter::cti_op_push_new_scope);
1084             emitPutVirtualRegister(currentInstruction[1].u.operand);
1085             NEXT_OPCODE(op_push_new_scope);
1086         }
1087         case op_catch: {
1088             emitGetCTIParam(STUB_ARGS_callFrame, callFrameRegister);
1089             emitPutVirtualRegister(currentInstruction[1].u.operand);
1090             NEXT_OPCODE(op_catch);
1091         }
1092         case op_jmp_scopes: {
1093             unsigned count = currentInstruction[1].u.operand;
1094             emitPutJITStubArgConstant(count, 1);
1095             emitCTICall(Interpreter::cti_op_jmp_scopes);
1096             unsigned target = currentInstruction[2].u.operand;
1097             addJump(jump(), target + 2);
1098             NEXT_OPCODE(op_jmp_scopes);
1099         }
1100         case op_put_by_index: {
1101             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
1102             emitPutJITStubArgConstant(currentInstruction[2].u.operand, 2);
1103             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 3, X86::ecx);
1104             emitCTICall(Interpreter::cti_op_put_by_index);
1105             NEXT_OPCODE(op_put_by_index);
1106         }
1107         case op_switch_imm: {
1108             unsigned tableIndex = currentInstruction[1].u.operand;
1109             unsigned defaultOffset = currentInstruction[2].u.operand;
1110             unsigned scrutinee = currentInstruction[3].u.operand;
1111
1112             // create jump table for switch destinations, track this switch statement.
1113             SimpleJumpTable* jumpTable = &m_codeBlock->immediateSwitchJumpTable(tableIndex);
1114             m_switches.append(SwitchRecord(jumpTable, m_bytecodeIndex, defaultOffset, SwitchRecord::Immediate));
1115             jumpTable->ctiOffsets.grow(jumpTable->branchOffsets.size());
1116
1117             emitPutJITStubArgFromVirtualRegister(scrutinee, 1, X86::ecx);
1118             emitPutJITStubArgConstant(tableIndex, 2);
1119             emitCTICall(Interpreter::cti_op_switch_imm);
1120             jump(X86::eax);
1121             NEXT_OPCODE(op_switch_imm);
1122         }
1123         case op_switch_char: {
1124             unsigned tableIndex = currentInstruction[1].u.operand;
1125             unsigned defaultOffset = currentInstruction[2].u.operand;
1126             unsigned scrutinee = currentInstruction[3].u.operand;
1127
1128             // create jump table for switch destinations, track this switch statement.
1129             SimpleJumpTable* jumpTable = &m_codeBlock->characterSwitchJumpTable(tableIndex);
1130             m_switches.append(SwitchRecord(jumpTable, m_bytecodeIndex, defaultOffset, SwitchRecord::Character));
1131             jumpTable->ctiOffsets.grow(jumpTable->branchOffsets.size());
1132
1133             emitPutJITStubArgFromVirtualRegister(scrutinee, 1, X86::ecx);
1134             emitPutJITStubArgConstant(tableIndex, 2);
1135             emitCTICall(Interpreter::cti_op_switch_char);
1136             jump(X86::eax);
1137             NEXT_OPCODE(op_switch_char);
1138         }
1139         case op_switch_string: {
1140             unsigned tableIndex = currentInstruction[1].u.operand;
1141             unsigned defaultOffset = currentInstruction[2].u.operand;
1142             unsigned scrutinee = currentInstruction[3].u.operand;
1143
1144             // create jump table for switch destinations, track this switch statement.
1145             StringJumpTable* jumpTable = &m_codeBlock->stringSwitchJumpTable(tableIndex);
1146             m_switches.append(SwitchRecord(jumpTable, m_bytecodeIndex, defaultOffset));
1147
1148             emitPutJITStubArgFromVirtualRegister(scrutinee, 1, X86::ecx);
1149             emitPutJITStubArgConstant(tableIndex, 2);
1150             emitCTICall(Interpreter::cti_op_switch_string);
1151             jump(X86::eax);
1152             NEXT_OPCODE(op_switch_string);
1153         }
1154         case op_del_by_val: {
1155             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
1156             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 2, X86::ecx);
1157             emitCTICall(Interpreter::cti_op_del_by_val);
1158             emitPutVirtualRegister(currentInstruction[1].u.operand);
1159             NEXT_OPCODE(op_del_by_val);
1160         }
1161         case op_put_getter: {
1162             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
1163             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
1164             emitPutJITStubArgConstant(ident, 2);
1165             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 3, X86::ecx);
1166             emitCTICall(Interpreter::cti_op_put_getter);
1167             NEXT_OPCODE(op_put_getter);
1168         }
1169         case op_put_setter: {
1170             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::ecx);
1171             Identifier* ident = &(m_codeBlock->identifier(currentInstruction[2].u.operand));
1172             emitPutJITStubArgConstant(ident, 2);
1173             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 3, X86::ecx);
1174             emitCTICall(Interpreter::cti_op_put_setter);
1175             NEXT_OPCODE(op_put_setter);
1176         }
1177         case op_new_error: {
1178             JSValue* message = m_codeBlock->unexpectedConstant(currentInstruction[3].u.operand);
1179             emitPutJITStubArgConstant(currentInstruction[2].u.operand, 1);
1180             emitPutJITStubArgConstant(message, 2);
1181             emitPutJITStubArgConstant(m_codeBlock->lineNumberForBytecodeOffset(m_bytecodeIndex), 3);
1182             emitCTICall(Interpreter::cti_op_new_error);
1183             emitPutVirtualRegister(currentInstruction[1].u.operand);
1184             NEXT_OPCODE(op_new_error);
1185         }
1186         case op_debug: {
1187             emitPutJITStubArgConstant(currentInstruction[1].u.operand, 1);
1188             emitPutJITStubArgConstant(currentInstruction[2].u.operand, 2);
1189             emitPutJITStubArgConstant(currentInstruction[3].u.operand, 3);
1190             emitCTICall(Interpreter::cti_op_debug);
1191             NEXT_OPCODE(op_debug);
1192         }
1193         case op_eq_null: {
1194             unsigned dst = currentInstruction[1].u.operand;
1195             unsigned src1 = currentInstruction[2].u.operand;
1196
1197             emitGetVirtualRegister(src1, X86::eax);
1198             Jump isImmediate = emitJumpIfNotJSCell(X86::eax);
1199
1200             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
1201             setnz32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), Imm32(MasqueradesAsUndefined), X86::eax);
1202
1203             Jump wasNotImmediate = jump();
1204
1205             isImmediate.link(this);
1206
1207             and32(Imm32(~JSImmediate::ExtendedTagBitUndefined), X86::eax);
1208             sete32(Imm32(JSImmediate::FullTagTypeNull), X86::eax);
1209
1210             wasNotImmediate.link(this);
1211
1212             emitTagAsBoolImmediate(X86::eax);
1213             emitPutVirtualRegister(dst);
1214
1215             NEXT_OPCODE(op_eq_null);
1216         }
1217         case op_neq_null: {
1218             unsigned dst = currentInstruction[1].u.operand;
1219             unsigned src1 = currentInstruction[2].u.operand;
1220
1221             emitGetVirtualRegister(src1, X86::eax);
1222             Jump isImmediate = emitJumpIfNotJSCell(X86::eax);
1223
1224             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::ecx);
1225             setz32(Address(X86::ecx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), Imm32(MasqueradesAsUndefined), X86::eax);
1226
1227             Jump wasNotImmediate = jump();
1228
1229             isImmediate.link(this);
1230
1231             and32(Imm32(~JSImmediate::ExtendedTagBitUndefined), X86::eax);
1232             setne32(Imm32(JSImmediate::FullTagTypeNull), X86::eax);
1233
1234             wasNotImmediate.link(this);
1235
1236             emitTagAsBoolImmediate(X86::eax);
1237             emitPutVirtualRegister(dst);
1238
1239             NEXT_OPCODE(op_neq_null);
1240         }
1241         case op_enter: {
1242             // Even though CTI doesn't use them, we initialize our constant
1243             // registers to zap stale pointers, to avoid unnecessarily prolonging
1244             // object lifetime and increasing GC pressure.
1245             size_t count = m_codeBlock->m_numVars + m_codeBlock->numberOfConstantRegisters();
1246             for (size_t j = 0; j < count; ++j)
1247                 emitInitRegister(j);
1248
1249             NEXT_OPCODE(op_enter);
1250         }
1251         case op_enter_with_activation: {
1252             // Even though CTI doesn't use them, we initialize our constant
1253             // registers to zap stale pointers, to avoid unnecessarily prolonging
1254             // object lifetime and increasing GC pressure.
1255             size_t count = m_codeBlock->m_numVars + m_codeBlock->numberOfConstantRegisters();
1256             for (size_t j = 0; j < count; ++j)
1257                 emitInitRegister(j);
1258
1259             emitCTICall(Interpreter::cti_op_push_activation);
1260             emitPutVirtualRegister(currentInstruction[1].u.operand);
1261
1262             NEXT_OPCODE(op_enter_with_activation);
1263         }
1264         case op_create_arguments: {
1265             if (m_codeBlock->m_numParameters == 1)
1266                 emitCTICall(Interpreter::cti_op_create_arguments_no_params);
1267             else
1268                 emitCTICall(Interpreter::cti_op_create_arguments);
1269             NEXT_OPCODE(op_create_arguments);
1270         }
1271         case op_convert_this: {
1272             emitGetVirtualRegister(currentInstruction[1].u.operand, X86::eax);
1273
1274             emitJumpSlowCaseIfNotJSCell(X86::eax);
1275             loadPtr(Address(X86::eax, FIELD_OFFSET(JSCell, m_structure)), X86::edx);
1276             addSlowCase(jnz32(Address(X86::edx, FIELD_OFFSET(Structure, m_typeInfo.m_flags)), Imm32(NeedsThisConversion)));
1277
1278             NEXT_OPCODE(op_convert_this);
1279         }
1280         case op_profile_will_call: {
1281             emitGetCTIParam(STUB_ARGS_profilerReference, X86::eax);
1282             Jump noProfiler = jzPtr(Address(X86::eax));
1283             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::eax);
1284             emitCTICall(Interpreter::cti_op_profile_will_call);
1285             noProfiler.link(this);
1286
1287             NEXT_OPCODE(op_profile_will_call);
1288         }
1289         case op_profile_did_call: {
1290             emitGetCTIParam(STUB_ARGS_profilerReference, X86::eax);
1291             Jump noProfiler = jzPtr(Address(X86::eax));
1292             emitPutJITStubArgFromVirtualRegister(currentInstruction[1].u.operand, 1, X86::eax);
1293             emitCTICall(Interpreter::cti_op_profile_did_call);
1294             noProfiler.link(this);
1295
1296             NEXT_OPCODE(op_profile_did_call);
1297         }
1298         case op_get_array_length:
1299         case op_get_by_id_chain:
1300         case op_get_by_id_generic:
1301         case op_get_by_id_proto:
1302         case op_get_by_id_proto_list:
1303         case op_get_by_id_self:
1304         case op_get_by_id_self_list:
1305         case op_get_string_length:
1306         case op_put_by_id_generic:
1307         case op_put_by_id_replace:
1308         case op_put_by_id_transition:
1309             ASSERT_NOT_REACHED();
1310         }
1311     }
1312
1313     ASSERT(propertyAccessInstructionIndex == m_codeBlock->numberOfStructureStubInfos());
1314     ASSERT(callLinkInfoIndex == m_codeBlock->numberOfCallLinkInfos());
1315
1316 #ifndef NDEBUG
1317     // reset this, in order to guard it's use with asserts
1318     m_bytecodeIndex = (unsigned)-1;
1319 #endif
1320 }
1321
1322
1323 void JIT::privateCompileLinkPass()
1324 {
1325     unsigned jmpTableCount = m_jmpTable.size();
1326     for (unsigned i = 0; i < jmpTableCount; ++i)
1327         m_jmpTable[i].from.linkTo(m_labels[m_jmpTable[i].toBytecodeIndex], this);
1328     m_jmpTable.clear();
1329 }
1330
1331 void JIT::privateCompileSlowCases()
1332 {
1333     Instruction* instructionsBegin = m_codeBlock->instructions().begin();
1334     unsigned propertyAccessInstructionIndex = 0;
1335     unsigned callLinkInfoIndex = 0;
1336
1337     for (Vector<SlowCaseEntry>::iterator iter = m_slowCases.begin(); iter != m_slowCases.end();) {
1338         // FIXME: enable peephole optimizations for slow cases when applicable
1339         killLastResultRegister();
1340
1341         m_bytecodeIndex = iter->to;
1342 #ifndef NDEBUG
1343         unsigned firstTo = m_bytecodeIndex;
1344 #endif
1345         Instruction* currentInstruction = instructionsBegin + m_bytecodeIndex;
1346
1347         switch (OpcodeID opcodeID = m_interpreter->getOpcodeID(currentInstruction->u.opcode)) {
1348         case op_convert_this: {
1349             linkSlowCase(iter);
1350             linkSlowCase(iter);
1351             emitPutJITStubArg(X86::eax, 1);
1352             emitCTICall(Interpreter::cti_op_convert_this);
1353             emitPutVirtualRegister(currentInstruction[1].u.operand);
1354             NEXT_OPCODE(op_convert_this);
1355         }
1356         case op_add: {
1357             unsigned dst = currentInstruction[1].u.operand;
1358             unsigned src1 = currentInstruction[2].u.operand;
1359             unsigned src2 = currentInstruction[3].u.operand;
1360             if (JSValue* value = getConstantImmediateNumericArg(src1)) {
1361                 Jump notImm = getSlowCase(iter);
1362                 linkSlowCase(iter);
1363                 sub32(Imm32(getDeTaggedConstantImmediate(value)), X86::eax);
1364                 notImm.link(this);
1365                 emitPutJITStubArgFromVirtualRegister(src1, 1, X86::ecx);
1366                 emitPutJITStubArg(X86::eax, 2);
1367                 emitCTICall(Interpreter::cti_op_add);
1368                 emitPutVirtualRegister(dst);
1369             } else if (JSValue* value = getConstantImmediateNumericArg(src2)) {
1370                 Jump notImm = getSlowCase(iter);
1371                 linkSlowCase(iter);
1372                 sub32(Imm32(getDeTaggedConstantImmediate(value)), X86::eax);
1373                 notImm.link(this);
1374                 emitPutJITStubArg(X86::eax, 1);
1375                 emitPutJITStubArgFromVirtualRegister(src2, 2, X86::ecx);
1376                 emitCTICall(Interpreter::cti_op_add);
1377                 emitPutVirtualRegister(dst);
1378             } else {
1379                 OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
1380                 ASSERT(types.first().mightBeNumber() && types.second().mightBeNumber());
1381                 compileBinaryArithOpSlowCase(op_add, iter, dst, src1, src2, types);
1382             }
1383
1384             NEXT_OPCODE(op_add);
1385         }
1386         case op_construct_verify: {
1387             linkSlowCase(iter);
1388             linkSlowCase(iter);
1389             emitGetVirtualRegister(currentInstruction[2].u.operand, X86::eax);
1390             emitPutVirtualRegister(currentInstruction[1].u.operand);
1391
1392             NEXT_OPCODE(op_construct_verify);
1393         }
1394         case op_get_by_val: {
1395             // The slow case that handles accesses to arrays (below) may jump back up to here. 
1396             Label beginGetByValSlow(this);
1397
1398             Jump notImm = getSlowCase(iter);
1399             linkSlowCase(iter);
1400             linkSlowCase(iter);
1401             emitFastArithIntToImmNoCheck(X86::edx);
1402             notImm.link(this);
1403             emitPutJITStubArg(X86::eax, 1);
1404             emitPutJITStubArg(X86::edx, 2);
1405             emitCTICall(Interpreter::cti_op_get_by_val);
1406             emitPutVirtualRegister(currentInstruction[1].u.operand);
1407             emitJumpSlowToHot(jump(), OPCODE_LENGTH(op_get_by_val));
1408
1409             // This is slow case that handles accesses to arrays above the fast cut-off.
1410             // First, check if this is an access to the vector
1411             linkSlowCase(iter);
1412             jae32(X86::edx, Address(X86::ecx, FIELD_OFFSET(ArrayStorage, m_vectorLength)), beginGetByValSlow);
1413
1414             // okay, missed the fast region, but it is still in the vector.  Get the value.
1415             loadPtr(BaseIndex(X86::ecx, X86::edx, ScalePtr, FIELD_OFFSET(ArrayStorage, m_vector[0])), X86::ecx);
1416             // Check whether the value loaded is zero; if so we need to return undefined.
1417             jzPtr(X86::ecx, beginGetByValSlow);
1418             move(X86::ecx, X86::eax);
1419             emitPutVirtualRegister(currentInstruction[1].u.operand, X86::eax);
1420
1421             NEXT_OPCODE(op_get_by_val);
1422         }
1423         case op_sub: {
1424             compileBinaryArithOpSlowCase(op_sub, iter, currentInstruction[1].u.operand, currentInstruction[2].u.operand, currentInstruction[3].u.operand, OperandTypes::fromInt(currentInstruction[4].u.operand));
1425             NEXT_OPCODE(op_sub);
1426         }
1427         case op_rshift: {
1428             unsigned src2 = currentInstruction[3].u.operand;
1429             linkSlowCase(iter);
1430             if (getConstantImmediateNumericArg(src2))
1431                 emitPutJITStubArgFromVirtualRegister(src2, 2, X86::ecx);
1432             else {
1433                 linkSlowCase(iter);
1434                 emitPutJITStubArg(X86::ecx, 2);
1435             }
1436
1437             emitPutJITStubArg(X86::eax, 1);
1438             emitCTICall(Interpreter::cti_op_rshift);
1439             emitPutVirtualRegister(currentInstruction[1].u.operand);
1440             NEXT_OPCODE(op_rshift);
1441         }
1442         case op_lshift: {
1443             Jump notImm1 = getSlowCase(iter);
1444             Jump notImm2 = getSlowCase(iter);
1445             linkSlowCase(iter);
1446             emitGetVirtualRegisters(currentInstruction[2].u.operand, X86::eax, currentInstruction[3].u.operand, X86::ecx);
1447             notImm1.link(this);
1448             notImm2.link(this);
1449             emitPutJITStubArg(X86::eax, 1);
1450             emitPutJITStubArg(X86::ecx, 2);
1451             emitCTICall(Interpreter::cti_op_lshift);
1452             emitPutVirtualRegister(currentInstruction[1].u.operand);
1453             NEXT_OPCODE(op_lshift);
1454         }
1455         case op_loop_if_less: {
1456             unsigned target = currentInstruction[3].u.operand;
1457             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
1458             if (src2imm) {
1459                 linkSlowCase(iter);
1460                 emitPutJITStubArg(X86::eax, 1);
1461                 emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 2, X86::ecx);
1462                 emitCTICall(Interpreter::cti_op_loop_if_less);
1463                 emitJumpSlowToHot(jnz32(X86::eax), target + 3);
1464             } else {
1465                 linkSlowCase(iter);
1466                 linkSlowCase(iter);
1467                 emitPutJITStubArg(X86::eax, 1);
1468                 emitPutJITStubArg(X86::edx, 2);
1469                 emitCTICall(Interpreter::cti_op_loop_if_less);
1470                 emitJumpSlowToHot(jnz32(X86::eax), target + 3);
1471             }
1472             NEXT_OPCODE(op_loop_if_less);
1473         }
1474         case op_put_by_id: {
1475             compilePutByIdSlowCase(currentInstruction[1].u.operand, &(m_codeBlock->identifier(currentInstruction[2].u.operand)), currentInstruction[3].u.operand, iter, propertyAccessInstructionIndex++);
1476             NEXT_OPCODE(op_put_by_id);
1477         }
1478         case op_get_by_id: {
1479             compileGetByIdSlowCase(currentInstruction[1].u.operand, currentInstruction[2].u.operand, &(m_codeBlock->identifier(currentInstruction[3].u.operand)), iter, propertyAccessInstructionIndex++);
1480             NEXT_OPCODE(op_get_by_id);
1481         }
1482         case op_loop_if_lesseq: {
1483             unsigned target = currentInstruction[3].u.operand;
1484             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
1485             if (src2imm) {
1486                 linkSlowCase(iter);
1487                 emitPutJITStubArg(X86::eax, 1);
1488                 emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 2, X86::ecx);
1489                 emitCTICall(Interpreter::cti_op_loop_if_lesseq);
1490                 emitJumpSlowToHot(jnz32(X86::eax), target + 3);
1491             } else {
1492                 linkSlowCase(iter);
1493                 linkSlowCase(iter);
1494                 emitPutJITStubArg(X86::eax, 1);
1495                 emitPutJITStubArg(X86::edx, 2);
1496                 emitCTICall(Interpreter::cti_op_loop_if_lesseq);
1497                 emitJumpSlowToHot(jnz32(X86::eax), target + 3);
1498             }
1499             NEXT_OPCODE(op_loop_if_lesseq);
1500         }
1501         case op_pre_inc: {
1502             unsigned srcDst = currentInstruction[1].u.operand;
1503             Jump notImm = getSlowCase(iter);
1504             linkSlowCase(iter);
1505             sub32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::eax);
1506             notImm.link(this);
1507             emitPutJITStubArg(X86::eax, 1);
1508             emitCTICall(Interpreter::cti_op_pre_inc);
1509             emitPutVirtualRegister(srcDst);
1510             NEXT_OPCODE(op_pre_inc);
1511         }
1512         case op_put_by_val: {
1513             // Normal slow cases - either is not an immediate imm, or is an array.
1514             Jump notImm = getSlowCase(iter);
1515             linkSlowCase(iter);
1516             linkSlowCase(iter);
1517             emitFastArithIntToImmNoCheck(X86::edx);
1518             notImm.link(this);
1519             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::ecx);
1520             emitPutJITStubArg(X86::eax, 1);
1521             emitPutJITStubArg(X86::edx, 2);
1522             emitPutJITStubArg(X86::ecx, 3);
1523             emitCTICall(Interpreter::cti_op_put_by_val);
1524             emitJumpSlowToHot(jump(), OPCODE_LENGTH(op_put_by_val));
1525
1526             // slow cases for immediate int accesses to arrays
1527             linkSlowCase(iter);
1528             linkSlowCase(iter);
1529             emitGetVirtualRegister(currentInstruction[3].u.operand, X86::ecx);
1530             emitPutJITStubArg(X86::eax, 1);
1531             emitPutJITStubArg(X86::edx, 2);
1532             emitPutJITStubArg(X86::ecx, 3);
1533             emitCTICall(Interpreter::cti_op_put_by_val_array);
1534
1535             NEXT_OPCODE(op_put_by_val);
1536         }
1537         case op_loop_if_true: {
1538             linkSlowCase(iter);
1539             emitPutJITStubArg(X86::eax, 1);
1540             emitCTICall(Interpreter::cti_op_jtrue);
1541             unsigned target = currentInstruction[2].u.operand;
1542             emitJumpSlowToHot(jnz32(X86::eax), target + 2);
1543             NEXT_OPCODE(op_loop_if_true);
1544         }
1545         case op_pre_dec: {
1546             unsigned srcDst = currentInstruction[1].u.operand;
1547             Jump notImm = getSlowCase(iter);
1548             linkSlowCase(iter);
1549             add32(Imm32(getDeTaggedConstantImmediate(JSImmediate::oneImmediate())), X86::eax);
1550             notImm.link(this);
1551             emitPutJITStubArg(X86::eax, 1);
1552             emitCTICall(Interpreter::cti_op_pre_dec);
1553             emitPutVirtualRegister(srcDst);
1554             NEXT_OPCODE(op_pre_dec);
1555         }
1556         case op_jnless: {
1557             unsigned target = currentInstruction[3].u.operand;
1558             JSValue* src2imm = getConstantImmediateNumericArg(currentInstruction[2].u.operand);
1559             if (src2imm) {
1560                 linkSlowCase(iter);
1561                 emitPutJITStubArg(X86::edx, 1);
1562                 emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 2, X86::ecx);
1563                 emitCTICall(Interpreter::cti_op_jless);
1564                 emitJumpSlowToHot(jz32(X86::eax), target + 3);
1565             } else {
1566                 linkSlowCase(iter);
1567                 linkSlowCase(iter);
1568                 emitPutJITStubArg(X86::eax, 1);
1569                 emitPutJITStubArg(X86::edx, 2);
1570                 emitCTICall(Interpreter::cti_op_jless);
1571                 emitJumpSlowToHot(jz32(X86::eax), target + 3);
1572             }
1573             NEXT_OPCODE(op_jnless);
1574         }
1575         case op_not: {
1576             linkSlowCase(iter);
1577             xorPtr(Imm32(JSImmediate::FullTagTypeBool), X86::eax);
1578             emitPutJITStubArg(X86::eax, 1);
1579             emitCTICall(Interpreter::cti_op_not);
1580             emitPutVirtualRegister(currentInstruction[1].u.operand);
1581             NEXT_OPCODE(op_not);
1582         }
1583         case op_jfalse: {
1584             linkSlowCase(iter);
1585             emitPutJITStubArg(X86::eax, 1);
1586             emitCTICall(Interpreter::cti_op_jtrue);
1587             unsigned target = currentInstruction[2].u.operand;
1588             emitJumpSlowToHot(jz32(X86::eax), target + 2); // inverted!
1589             NEXT_OPCODE(op_jfalse);
1590         }
1591         case op_post_inc: {
1592             unsigned srcDst = currentInstruction[2].u.operand;
1593             linkSlowCase(iter);
1594             linkSlowCase(iter);
1595             emitPutJITStubArg(X86::eax, 1);
1596             emitCTICall(Interpreter::cti_op_post_inc);
1597             emitPutVirtualRegister(srcDst, X86::edx);
1598             emitPutVirtualRegister(currentInstruction[1].u.operand);
1599             NEXT_OPCODE(op_post_inc);
1600         }
1601         case op_bitnot: {
1602             linkSlowCase(iter);
1603             emitPutJITStubArg(X86::eax, 1);
1604             emitCTICall(Interpreter::cti_op_bitnot);
1605             emitPutVirtualRegister(currentInstruction[1].u.operand);
1606             NEXT_OPCODE(op_bitnot);
1607         }
1608         case op_bitand: {
1609             linkSlowCase(iter);
1610             unsigned src1 = currentInstruction[2].u.operand;
1611             unsigned src2 = currentInstruction[3].u.operand;
1612             unsigned dst = currentInstruction[1].u.operand;
1613             if (getConstantImmediateNumericArg(src1)) {
1614                 emitPutJITStubArgFromVirtualRegister(src1, 1, X86::ecx);
1615                 emitPutJITStubArg(X86::eax, 2);
1616                 emitCTICall(Interpreter::cti_op_bitand);
1617                 emitPutVirtualRegister(dst);
1618             } else if (getConstantImmediateNumericArg(src2)) {
1619                 emitPutJITStubArg(X86::eax, 1);
1620                 emitPutJITStubArgFromVirtualRegister(src2, 2, X86::ecx);
1621                 emitCTICall(Interpreter::cti_op_bitand);
1622                 emitPutVirtualRegister(dst);
1623             } else {
1624                 emitPutJITStubArgFromVirtualRegister(src1, 1, X86::ecx);
1625                 emitPutJITStubArg(X86::edx, 2);
1626                 emitCTICall(Interpreter::cti_op_bitand);
1627                 emitPutVirtualRegister(dst);
1628             }
1629             NEXT_OPCODE(op_bitand);
1630         }
1631         case op_jtrue: {
1632             linkSlowCase(iter);
1633             emitPutJITStubArg(X86::eax, 1);
1634             emitCTICall(Interpreter::cti_op_jtrue);
1635             unsigned target = currentInstruction[2].u.operand;
1636             emitJumpSlowToHot(jnz32(X86::eax), target + 2);
1637             NEXT_OPCODE(op_jtrue);
1638         }
1639         case op_post_dec: {
1640             unsigned srcDst = currentInstruction[2].u.operand;
1641             linkSlowCase(iter);
1642             linkSlowCase(iter);
1643             emitPutJITStubArg(X86::eax, 1);
1644             emitCTICall(Interpreter::cti_op_post_dec);
1645             emitPutVirtualRegister(srcDst, X86::edx);
1646             emitPutVirtualRegister(currentInstruction[1].u.operand);
1647             NEXT_OPCODE(op_post_dec);
1648         }
1649         case op_bitxor: {
1650             linkSlowCase(iter);
1651             emitPutJITStubArg(X86::eax, 1);
1652             emitPutJITStubArg(X86::edx, 2);
1653             emitCTICall(Interpreter::cti_op_bitxor);
1654             emitPutVirtualRegister(currentInstruction[1].u.operand);
1655             NEXT_OPCODE(op_bitxor);
1656         }
1657         case op_bitor: {
1658             linkSlowCase(iter);
1659             emitPutJITStubArg(X86::eax, 1);
1660             emitPutJITStubArg(X86::edx, 2);
1661             emitCTICall(Interpreter::cti_op_bitor);
1662             emitPutVirtualRegister(currentInstruction[1].u.operand);
1663             NEXT_OPCODE(op_bitor);
1664         }
1665         case op_eq: {
1666             linkSlowCase(iter);
1667             emitPutJITStubArg(X86::eax, 1);
1668             emitPutJITStubArg(X86::edx, 2);
1669             emitCTICall(Interpreter::cti_op_eq);
1670             emitPutVirtualRegister(currentInstruction[1].u.operand);
1671             NEXT_OPCODE(op_eq);
1672         }
1673         case op_neq: {
1674             linkSlowCase(iter);
1675             emitPutJITStubArg(X86::eax, 1);
1676             emitPutJITStubArg(X86::edx, 2);
1677             emitCTICall(Interpreter::cti_op_neq);
1678             emitPutVirtualRegister(currentInstruction[1].u.operand);
1679             NEXT_OPCODE(op_neq);
1680         }
1681         case op_stricteq: {
1682             linkSlowCase(iter);
1683             linkSlowCase(iter);
1684             linkSlowCase(iter);
1685             emitPutJITStubArg(X86::eax, 1);
1686             emitPutJITStubArg(X86::edx, 2);
1687             emitCTICall(Interpreter::cti_op_stricteq);
1688             emitPutVirtualRegister(currentInstruction[1].u.operand);
1689             NEXT_OPCODE(op_stricteq);
1690         }
1691         case op_nstricteq: {
1692             linkSlowCase(iter);
1693             linkSlowCase(iter);
1694             linkSlowCase(iter);
1695             emitPutJITStubArg(X86::eax, 1);
1696             emitPutJITStubArg(X86::edx, 2);
1697             emitCTICall(Interpreter::cti_op_nstricteq);
1698             emitPutVirtualRegister(currentInstruction[1].u.operand);
1699             NEXT_OPCODE(op_nstricteq);
1700         }
1701         case op_instanceof: {
1702             linkSlowCase(iter);
1703             linkSlowCase(iter);
1704             linkSlowCase(iter);
1705             emitPutJITStubArgFromVirtualRegister(currentInstruction[2].u.operand, 1, X86::ecx);
1706             emitPutJITStubArgFromVirtualRegister(currentInstruction[3].u.operand, 2, X86::ecx);
1707             emitPutJITStubArgFromVirtualRegister(currentInstruction[4].u.operand, 3, X86::ecx);
1708             emitCTICall(Interpreter::cti_op_instanceof);
1709             emitPutVirtualRegister(currentInstruction[1].u.operand);
1710             NEXT_OPCODE(op_instanceof);
1711         }
1712         case op_mod: {
1713             Jump notImm1 = getSlowCase(iter);
1714             Jump notImm2 = getSlowCase(iter);
1715             linkSlowCase(iter);
1716             emitFastArithReTagImmediate(X86::eax);
1717             emitFastArithReTagImmediate(X86::ecx);
1718             notImm1.link(this);
1719             notImm2.link(this);
1720             emitPutJITStubArg(X86::eax, 1);
1721             emitPutJITStubArg(X86::ecx, 2);
1722             emitCTICall(Interpreter::cti_op_mod);
1723             emitPutVirtualRegister(currentInstruction[1].u.operand);
1724             NEXT_OPCODE(op_mod);
1725         }
1726         case op_mul: {
1727             int dst = currentInstruction[1].u.operand;
1728             int src1 = currentInstruction[2].u.operand;
1729             int src2 = currentInstruction[3].u.operand;
1730             JSValue* src1Value = getConstantImmediateNumericArg(src1);
1731             JSValue* src2Value = getConstantImmediateNumericArg(src2);
1732             int32_t value;
1733             if (src1Value && ((value = JSImmediate::intValue(src1Value)) > 0)) {
1734                 linkSlowCase(iter);
1735                 linkSlowCase(iter);
1736                 // There is an extra slow case for (op1 * -N) or (-N * op2), to check for 0 since this should produce a result of -0.
1737                 emitPutJITStubArgFromVirtualRegister(src1, 1, X86::ecx);
1738                 emitPutJITStubArgFromVirtualRegister(src2, 2, X86::ecx);
1739                 emitCTICall(Interpreter::cti_op_mul);
1740                 emitPutVirtualRegister(dst);
1741             } else if (src2Value && ((value = JSImmediate::intValue(src2Value)) > 0)) {
1742                 linkSlowCase(iter);
1743                 linkSlowCase(iter);
1744                 // There is an extra slow case for (op1 * -N) or (-N * op2), to check for 0 since this should produce a result of -0.
1745                 emitPutJITStubArgFromVirtualRegister(src1, 1, X86::ecx);
1746                 emitPutJITStubArgFromVirtualRegister(src2, 2, X86::ecx);
1747                 emitCTICall(Interpreter::cti_op_mul);
1748                 emitPutVirtualRegister(dst);
1749             } else
1750                 compileBinaryArithOpSlowCase(op_mul, iter, dst, src1, src2, OperandTypes::fromInt(currentInstruction[4].u.operand));
1751             NEXT_OPCODE(op_mul);
1752         }
1753
1754         case op_call: {
1755             compileOpCallSlowCase(currentInstruction, iter, callLinkInfoIndex++, opcodeID);
1756             NEXT_OPCODE(op_call);
1757         }
1758         case op_call_eval: {
1759             compileOpCallSlowCase(currentInstruction, iter, callLinkInfoIndex++, opcodeID);
1760             NEXT_OPCODE(op_call_eval);
1761         }
1762         case op_construct: {
1763             compileOpCallSlowCase(currentInstruction, iter, callLinkInfoIndex++, opcodeID);
1764             NEXT_OPCODE(op_construct);
1765         }
1766         case op_to_jsnumber: {
1767             linkSlowCaseIfNotJSCell(iter, currentInstruction[2].u.operand);
1768             linkSlowCase(iter);
1769
1770             emitPutJITStubArg(X86::eax, 1);
1771             emitCTICall(Interpreter::cti_op_to_jsnumber);
1772
1773             emitPutVirtualRegister(currentInstruction[1].u.operand);
1774             NEXT_OPCODE(op_to_jsnumber);
1775         }
1776
1777         default:
1778             ASSERT_NOT_REACHED();
1779         }
1780
1781         ASSERT_WITH_MESSAGE(iter == m_slowCases.end() || firstTo != iter->to,"Not enough jumps linked in slow case codegen.");
1782         ASSERT_WITH_MESSAGE(firstTo == (iter - 1)->to, "Too many jumps linked in slow case codegen.");
1783
1784         emitJumpSlowToHot(jump(), 0);
1785     }
1786
1787 #if ENABLE(JIT_OPTIMIZE_PROPERTY_ACCESS)
1788     ASSERT(propertyAccessInstructionIndex == m_codeBlock->numberOfStructureStubInfos());
1789 #endif
1790     ASSERT(callLinkInfoIndex == m_codeBlock->numberOfCallLinkInfos());
1791
1792 #ifndef NDEBUG
1793     // reset this, in order to guard it's use with asserts
1794     m_bytecodeIndex = (unsigned)-1;
1795 #endif
1796 }
1797
1798 void JIT::privateCompile()
1799 {
1800 #if ENABLE(CODEBLOCK_SAMPLING)
1801         storePtr(ImmPtr(m_codeBlock), m_interpreter->sampler()->codeBlockSlot());
1802 #endif
1803 #if ENABLE(OPCODE_SAMPLING)
1804         store32(Imm32(m_interpreter->sampler()->encodeSample(m_codeBlock->instructions().begin())), m_interpreter->sampler()->sampleSlot());
1805 #endif
1806
1807     // Could use a pop_m, but would need to offset the following instruction if so.
1808     pop(X86::ecx);
1809     emitPutToCallFrameHeader(X86::ecx, RegisterFile::ReturnPC);
1810
1811     Jump slowRegisterFileCheck;
1812     Label afterRegisterFileCheck;
1813     if (m_codeBlock->codeType() == FunctionCode) {
1814         // In the case of a fast linked call, we do not set this up in the caller.
1815         emitPutImmediateToCallFrameHeader(m_codeBlock, RegisterFile::CodeBlock);
1816
1817         emitGetCTIParam(STUB_ARGS_registerFile, X86::eax);
1818         addPtr(Imm32(m_codeBlock->m_numCalleeRegisters * sizeof(Register)), callFrameRegister, X86::edx);
1819         
1820         slowRegisterFileCheck = jg32(X86::edx, Address(X86::eax, FIELD_OFFSET(RegisterFile, m_end)));
1821         afterRegisterFileCheck = label();
1822     }
1823
1824     privateCompileMainPass();
1825     privateCompileLinkPass();
1826     privateCompileSlowCases();
1827
1828     if (m_codeBlock->codeType() == FunctionCode) {
1829         slowRegisterFileCheck.link(this);
1830         m_bytecodeIndex = 0; // emitCTICall will add to the map, but doesn't actually need this...
1831         emitCTICall(Interpreter::cti_register_file_check);
1832 #ifndef NDEBUG
1833         // reset this, in order to guard it's use with asserts
1834         m_bytecodeIndex = (unsigned)-1;
1835 #endif
1836         jump(afterRegisterFileCheck);
1837     }
1838
1839     ASSERT(m_jmpTable.isEmpty());
1840
1841     RefPtr<ExecutablePool> allocator = m_globalData->poolForSize(m_assembler.size());
1842     void* code = m_assembler.executableCopy(allocator.get());
1843     JITCodeRef codeRef(code, allocator);
1844  
1845     PatchBuffer patchBuffer(code);
1846
1847     // Translate vPC offsets into addresses in JIT generated code, for switch tables.
1848     for (unsigned i = 0; i < m_switches.size(); ++i) {
1849         SwitchRecord record = m_switches[i];
1850         unsigned bytecodeIndex = record.bytecodeIndex;
1851
1852         if (record.type != SwitchRecord::String) {
1853             ASSERT(record.type == SwitchRecord::Immediate || record.type == SwitchRecord::Character); 
1854             ASSERT(record.jumpTable.simpleJumpTable->branchOffsets.size() == record.jumpTable.simpleJumpTable->ctiOffsets.size());
1855
1856             record.jumpTable.simpleJumpTable->ctiDefault = patchBuffer.addressOf(m_labels[bytecodeIndex + 3 + record.defaultOffset]);
1857
1858             for (unsigned j = 0; j < record.jumpTable.simpleJumpTable->branchOffsets.size(); ++j) {
1859                 unsigned offset = record.jumpTable.simpleJumpTable->branchOffsets[j];
1860                 record.jumpTable.simpleJumpTable->ctiOffsets[j] = offset ? patchBuffer.addressOf(m_labels[bytecodeIndex + 3 + offset]) : record.jumpTable.simpleJumpTable->ctiDefault;
1861             }
1862         } else {
1863             ASSERT(record.type == SwitchRecord::String);
1864
1865             record.jumpTable.stringJumpTable->ctiDefault = patchBuffer.addressOf(m_labels[bytecodeIndex + 3 + record.defaultOffset]);
1866
1867             StringJumpTable::StringOffsetTable::iterator end = record.jumpTable.stringJumpTable->offsetTable.end();            
1868             for (StringJumpTable::StringOffsetTable::iterator it = record.jumpTable.stringJumpTable->offsetTable.begin(); it != end; ++it) {
1869                 unsigned offset = it->second.branchOffset;
1870                 it->second.ctiOffset = offset ? patchBuffer.addressOf(m_labels[bytecodeIndex + 3 + offset]) : record.jumpTable.stringJumpTable->ctiDefault;
1871             }
1872         }
1873     }
1874
1875     for (size_t i = 0; i < m_codeBlock->numberOfExceptionHandlers(); ++i) {
1876         HandlerInfo& handler = m_codeBlock->exceptionHandler(i);
1877         handler.nativeCode = patchBuffer.addressOf(m_labels[handler.target]);
1878     }
1879
1880     m_codeBlock->pcVector().reserveCapacity(m_calls.size());
1881     for (Vector<CallRecord>::iterator iter = m_calls.begin(); iter != m_calls.end(); ++iter) {
1882         if (iter->to)
1883             patchBuffer.link(iter->from, iter->to);
1884         m_codeBlock->pcVector().append(PC(patchBuffer.addressOf(iter->from), iter->bytecodeIndex));
1885     }
1886
1887     // Link absolute addresses for jsr
1888     for (Vector<JSRInfo>::iterator iter = m_jsrSites.begin(); iter != m_jsrSites.end(); ++iter)
1889         patchBuffer.setPtr(iter->storeLocation, patchBuffer.addressOf(iter->target));
1890
1891     for (unsigned i = 0; i < m_codeBlock->numberOfStructureStubInfos(); ++i) {
1892         StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
1893 #if ENABLE(JIT_OPTIMIZE_PROPERTY_ACCESS)
1894         info.callReturnLocation = patchBuffer.addressOf(m_propertyAccessCompilationInfo[i].callReturnLocation);
1895         info.hotPathBegin = patchBuffer.addressOf(m_propertyAccessCompilationInfo[i].hotPathBegin);
1896 #else
1897         info.callReturnLocation = 0;
1898         info.hotPathBegin = 0;
1899 #endif
1900     }
1901     for (unsigned i = 0; i < m_codeBlock->numberOfCallLinkInfos(); ++i) {
1902         CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
1903 #if ENABLE(JIT_OPTIMIZE_CALL)
1904         info.callReturnLocation = patchBuffer.addressOf(m_callStructureStubCompilationInfo[i].callReturnLocation);
1905         info.hotPathBegin = patchBuffer.addressOf(m_callStructureStubCompilationInfo[i].hotPathBegin);
1906         info.hotPathOther = patchBuffer.addressOf(m_callStructureStubCompilationInfo[i].hotPathOther);
1907         info.coldPathOther = patchBuffer.addressOf(m_callStructureStubCompilationInfo[i].coldPathOther);
1908 #else
1909         info.callReturnLocation = 0;
1910         info.hotPathBegin = 0;
1911         info.hotPathOther = 0;
1912         info.coldPathOther = 0;
1913 #endif
1914     }
1915
1916     m_codeBlock->setJITCode(codeRef);
1917 }
1918
1919 void JIT::privateCompileCTIMachineTrampolines()
1920 {
1921 #if ENABLE(JIT_OPTIMIZE_PROPERTY_ACCESS)
1922     // (1) The first function provides fast property access for array length
1923     Label arrayLengthBegin = align();
1924
1925     // Check eax is an array
1926     Jump array_failureCases1 = emitJumpIfNotJSCell(X86::eax);
1927     Jump array_failureCases2 = jnePtr(Address(X86::eax), ImmPtr(m_interpreter->m_jsArrayVptr));
1928
1929     // Checks out okay! - get the length from the storage
1930     loadPtr(Address(X86::eax, FIELD_OFFSET(JSArray, m_storage)), X86::eax);
1931     load32(Address(X86::eax, FIELD_OFFSET(ArrayStorage, m_length)), X86::eax);
1932
1933     Jump array_failureCases3 = ja32(X86::eax, Imm32(JSImmediate::maxImmediateInt));
1934
1935     add32(X86::eax, X86::eax);
1936     add32(Imm32(1), X86::eax);
1937     
1938     ret();
1939
1940     // (2) The second function provides fast property access for string length
1941     Label stringLengthBegin = align();
1942
1943     // Check eax is a string
1944     Jump string_failureCases1 = emitJumpIfNotJSCell(X86::eax);
1945     Jump string_failureCases2 = jnePtr(Address(X86::eax), ImmPtr(m_interpreter->m_jsStringVptr));
1946
1947     // Checks out okay! - get the length from the Ustring.
1948     loadPtr(Address(X86::eax, FIELD_OFFSET(JSString, m_value) + FIELD_OFFSET(UString, m_rep)), X86::eax);
1949     load32(Address(X86::eax, FIELD_OFFSET(UString::Rep, len)), X86::eax);
1950
1951     Jump string_failureCases3 = ja32(X86::eax, Imm32(JSImmediate::maxImmediateInt));
1952
1953     add32(X86::eax, X86::eax);
1954     add32(Imm32(1), X86::eax);
1955     
1956     ret();
1957 #endif
1958
1959     // (3) Trampolines for the slow cases of op_call / op_call_eval / op_construct.
1960     
1961     Label virtualCallPreLinkBegin = align();
1962
1963     // Load the callee CodeBlock* into eax
1964     loadPtr(Address(X86::ecx, FIELD_OFFSET(JSFunction, m_body)), X86::eax);
1965     loadPtr(Address(X86::eax, FIELD_OFFSET(FunctionBodyNode, m_code)), X86::eax);
1966     Jump hasCodeBlock1 = jnzPtr(X86::eax);
1967     pop(X86::ebx);
1968     restoreArgumentReference();
1969     Jump callJSFunction1 = call();
1970     emitGetJITStubArg(1, X86::ecx);
1971     emitGetJITStubArg(3, X86::edx);
1972     push(X86::ebx);
1973     hasCodeBlock1.link(this);
1974
1975     // Check argCount matches callee arity.
1976     Jump arityCheckOkay1 = je32(Address(X86::eax, FIELD_OFFSET(CodeBlock, m_numParameters)), X86::edx);
1977     pop(X86::ebx);
1978     emitPutJITStubArg(X86::ebx, 2);
1979     emitPutJITStubArg(X86::eax, 4);
1980     restoreArgumentReference();
1981     Jump callArityCheck1 = call();
1982     move(X86::edx, callFrameRegister);
1983     emitGetJITStubArg(1, X86::ecx);
1984     emitGetJITStubArg(3, X86::edx);
1985     push(X86::ebx);
1986     arityCheckOkay1.link(this);
1987     
1988     compileOpCallInitializeCallFrame();
1989
1990     pop(X86::ebx);
1991     emitPutJITStubArg(X86::ebx, 2);
1992     restoreArgumentReference();
1993     Jump callDontLazyLinkCall = call();
1994     push(X86::ebx);
1995
1996     jump(X86::eax);
1997
1998     Label virtualCallLinkBegin = align();
1999
2000     // Load the callee CodeBlock* into eax
2001     loadPtr(Address(X86::ecx, FIELD_OFFSET(JSFunction, m_body)), X86::eax);
2002     loadPtr(Address(X86::eax, FIELD_OFFSET(FunctionBodyNode, m_code)), X86::eax);
2003     Jump hasCodeBlock2 = jnzPtr(X86::eax);
2004     pop(X86::ebx);
2005     restoreArgumentReference();
2006     Jump callJSFunction2 = call();
2007     emitGetJITStubArg(1, X86::ecx);
2008     emitGetJITStubArg(3, X86::edx);
2009     push(X86::ebx);
2010     hasCodeBlock2.link(this);
2011
2012     // Check argCount matches callee arity.
2013     Jump arityCheckOkay2 = je32(Address(X86::eax, FIELD_OFFSET(CodeBlock, m_numParameters)), X86::edx);
2014     pop(X86::ebx);
2015     emitPutJITStubArg(X86::ebx, 2);
2016     emitPutJITStubArg(X86::eax, 4);
2017     restoreArgumentReference();
2018     Jump callArityCheck2 = call();
2019     move(X86::edx, callFrameRegister);
2020     emitGetJITStubArg(1, X86::ecx);
2021     emitGetJITStubArg(3, X86::edx);
2022     push(X86::ebx);
2023     arityCheckOkay2.link(this);
2024
2025     compileOpCallInitializeCallFrame();
2026
2027     pop(X86::ebx);
2028     emitPutJITStubArg(X86::ebx, 2);
2029     restoreArgumentReference();
2030     Jump callLazyLinkCall = call();
2031     push(X86::ebx);
2032
2033     jump(X86::eax);
2034
2035     Label virtualCallBegin = align();
2036
2037     // Load the callee CodeBlock* into eax
2038     loadPtr(Address(X86::ecx, FIELD_OFFSET(JSFunction, m_body)), X86::eax);
2039     loadPtr(Address(X86::eax, FIELD_OFFSET(FunctionBodyNode, m_code)), X86::eax);
2040     Jump hasCodeBlock3 = jnzPtr(X86::eax);
2041     pop(X86::ebx);
2042     restoreArgumentReference();
2043     Jump callJSFunction3 = call();
2044     emitGetJITStubArg(1, X86::ecx);
2045     emitGetJITStubArg(3, X86::edx);
2046     push(X86::ebx);
2047     hasCodeBlock3.link(this);
2048
2049     // Check argCount matches callee arity.
2050     Jump arityCheckOkay3 = je32(Address(X86::eax, FIELD_OFFSET(CodeBlock, m_numParameters)), X86::edx);
2051     pop(X86::ebx);
2052     emitPutJITStubArg(X86::ebx, 2);
2053     emitPutJITStubArg(X86::eax, 4);
2054     restoreArgumentReference();
2055     Jump callArityCheck3 = call();
2056     move(X86::edx, callFrameRegister);
2057     emitGetJITStubArg(1, X86::ecx);
2058     emitGetJITStubArg(3, X86::edx);
2059     push(X86::ebx);
2060     arityCheckOkay3.link(this);
2061
2062     compileOpCallInitializeCallFrame();
2063
2064     // load ctiCode from the new codeBlock.
2065     loadPtr(Address(X86::eax, FIELD_OFFSET(CodeBlock, m_jitCode)), X86::eax);
2066
2067     jump(X86::eax);
2068
2069     // All trampolines constructed! copy the code, link up calls, and set the pointers on the Machine object.
2070     m_interpreter->m_executablePool = m_globalData->poolForSize(m_assembler.size());
2071     void* code = m_assembler.executableCopy(m_interpreter->m_executablePool.get());
2072     PatchBuffer patchBuffer(code);
2073
2074 #if ENABLE(JIT_OPTIMIZE_PROPERTY_ACCESS)
2075     patchBuffer.link(array_failureCases1, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_array_fail));
2076     patchBuffer.link(array_failureCases2, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_array_fail));
2077     patchBuffer.link(array_failureCases3, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_array_fail));
2078     patchBuffer.link(string_failureCases1, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_string_fail));
2079     patchBuffer.link(string_failureCases2, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_string_fail));
2080     patchBuffer.link(string_failureCases3, reinterpret_cast<void*>(Interpreter::cti_op_get_by_id_string_fail));
2081
2082     m_interpreter->m_ctiArrayLengthTrampoline = patchBuffer.addressOf(arrayLengthBegin);
2083     m_interpreter->m_ctiStringLengthTrampoline = patchBuffer.addressOf(stringLengthBegin);
2084 #endif
2085     patchBuffer.link(callArityCheck1, reinterpret_cast<void*>(Interpreter::cti_op_call_arityCheck));
2086     patchBuffer.link(callArityCheck2, reinterpret_cast<void*>(Interpreter::cti_op_call_arityCheck));
2087     patchBuffer.link(callArityCheck3, reinterpret_cast<void*>(Interpreter::cti_op_call_arityCheck));
2088     patchBuffer.link(callJSFunction1, reinterpret_cast<void*>(Interpreter::cti_op_call_JSFunction));
2089     patchBuffer.link(callJSFunction2, reinterpret_cast<void*>(Interpreter::cti_op_call_JSFunction));
2090     patchBuffer.link(callJSFunction3, reinterpret_cast<void*>(Interpreter::cti_op_call_JSFunction));
2091     patchBuffer.link(callDontLazyLinkCall, reinterpret_cast<void*>(Interpreter::cti_vm_dontLazyLinkCall));
2092     patchBuffer.link(callLazyLinkCall, reinterpret_cast<void*>(Interpreter::cti_vm_lazyLinkCall));
2093
2094     m_interpreter->m_ctiVirtualCallPreLink = patchBuffer.addressOf(virtualCallPreLinkBegin);
2095     m_interpreter->m_ctiVirtualCallLink = patchBuffer.addressOf(virtualCallLinkBegin);
2096     m_interpreter->m_ctiVirtualCall = patchBuffer.addressOf(virtualCallBegin);
2097 }
2098
2099 void JIT::emitGetVariableObjectRegister(RegisterID variableObject, int index, RegisterID dst)
2100 {
2101     loadPtr(Address(variableObject, FIELD_OFFSET(JSVariableObject, d)), dst);
2102     loadPtr(Address(dst, FIELD_OFFSET(JSVariableObject::JSVariableObjectData, registers)), dst);
2103     loadPtr(Address(dst, index * sizeof(Register)), dst);
2104 }
2105
2106 void JIT::emitPutVariableObjectRegister(RegisterID src, RegisterID variableObject, int index)
2107 {
2108     loadPtr(Address(variableObject, FIELD_OFFSET(JSVariableObject, d)), variableObject);
2109     loadPtr(Address(variableObject, FIELD_OFFSET(JSVariableObject::JSVariableObjectData, registers)), variableObject);
2110     storePtr(src, Address(variableObject, index * sizeof(Register)));
2111 }
2112
2113 } // namespace JSC
2114
2115 #endif // ENABLE(JIT)