Harden JSC a bit with RELEASE_ASSERT
[WebKit-https.git] / Source / JavaScriptCore / jit / JIT.cpp
1 /*
2  * Copyright (C) 2008, 2009, 2012, 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(JIT)
29 #include "JIT.h"
30
31 // This probably does not belong here; adding here for now as a quick Windows build fix.
32 #if ENABLE(ASSEMBLER) && CPU(X86) && !OS(MAC_OS_X)
33 #include "MacroAssembler.h"
34 JSC::MacroAssemblerX86Common::SSE2CheckState JSC::MacroAssemblerX86Common::s_sse2CheckState = NotCheckedSSE2;
35 #endif
36
37 #include "CodeBlock.h"
38 #include <wtf/CryptographicallyRandomNumber.h>
39 #include "DFGNode.h" // for DFG_SUCCESS_STATS
40 #include "Interpreter.h"
41 #include "JITInlines.h"
42 #include "JITStubCall.h"
43 #include "JSArray.h"
44 #include "JSFunction.h"
45 #include "LinkBuffer.h"
46 #include "Operations.h"
47 #include "RepatchBuffer.h"
48 #include "ResultType.h"
49 #include "SamplingTool.h"
50
51 using namespace std;
52
53 namespace JSC {
54
55 void ctiPatchNearCallByReturnAddress(CodeBlock* codeblock, ReturnAddressPtr returnAddress, MacroAssemblerCodePtr newCalleeFunction)
56 {
57     RepatchBuffer repatchBuffer(codeblock);
58     repatchBuffer.relinkNearCallerToTrampoline(returnAddress, newCalleeFunction);
59 }
60
61 void ctiPatchCallByReturnAddress(CodeBlock* codeblock, ReturnAddressPtr returnAddress, MacroAssemblerCodePtr newCalleeFunction)
62 {
63     RepatchBuffer repatchBuffer(codeblock);
64     repatchBuffer.relinkCallerToTrampoline(returnAddress, newCalleeFunction);
65 }
66
67 void ctiPatchCallByReturnAddress(CodeBlock* codeblock, ReturnAddressPtr returnAddress, FunctionPtr newCalleeFunction)
68 {
69     RepatchBuffer repatchBuffer(codeblock);
70     repatchBuffer.relinkCallerToFunction(returnAddress, newCalleeFunction);
71 }
72
73 JIT::JIT(JSGlobalData* globalData, CodeBlock* codeBlock)
74     : m_interpreter(globalData->interpreter)
75     , m_globalData(globalData)
76     , m_codeBlock(codeBlock)
77     , m_labels(codeBlock ? codeBlock->numberOfInstructions() : 0)
78     , m_bytecodeOffset((unsigned)-1)
79     , m_propertyAccessInstructionIndex(UINT_MAX)
80     , m_byValInstructionIndex(UINT_MAX)
81     , m_globalResolveInfoIndex(UINT_MAX)
82     , m_callLinkInfoIndex(UINT_MAX)
83 #if USE(JSVALUE32_64)
84     , m_jumpTargetIndex(0)
85     , m_mappedBytecodeOffset((unsigned)-1)
86     , m_mappedVirtualRegisterIndex(JSStack::ReturnPC)
87     , m_mappedTag((RegisterID)-1)
88     , m_mappedPayload((RegisterID)-1)
89 #else
90     , m_lastResultBytecodeRegister(std::numeric_limits<int>::max())
91     , m_jumpTargetsPosition(0)
92 #endif
93 #if USE(OS_RANDOMNESS)
94     , m_randomGenerator(cryptographicallyRandomNumber())
95 #else
96     , m_randomGenerator(static_cast<unsigned>(randomNumber() * 0xFFFFFFF))
97 #endif
98 #if ENABLE(VALUE_PROFILER)
99     , m_canBeOptimized(false)
100     , m_shouldEmitProfiling(false)
101 #endif
102 {
103 }
104
105 #if ENABLE(DFG_JIT)
106 void JIT::emitOptimizationCheck(OptimizationCheckKind kind)
107 {
108     if (!canBeOptimized())
109         return;
110     
111     Jump skipOptimize = branchAdd32(Signed, TrustedImm32(kind == LoopOptimizationCheck ? Options::executionCounterIncrementForLoop() : Options::executionCounterIncrementForReturn()), AbsoluteAddress(m_codeBlock->addressOfJITExecuteCounter()));
112     JITStubCall stubCall(this, cti_optimize);
113     stubCall.addArgument(TrustedImm32(m_bytecodeOffset));
114     if (kind == EnterOptimizationCheck)
115         ASSERT(!m_bytecodeOffset);
116     stubCall.call();
117     skipOptimize.link(this);
118 }
119 #endif
120
121 #if CPU(X86)
122 void JIT::emitTimeoutCheck()
123 {
124     Jump skipTimeout = branchSub32(NonZero, TrustedImm32(1), AbsoluteAddress(&m_globalData->m_timeoutCount));
125     JITStubCall stubCall(this, cti_timeout_check);
126     stubCall.addArgument(regT1, regT0); // save last result registers.
127     stubCall.call(regT0);
128     store32(regT0, &m_globalData->m_timeoutCount);
129     stubCall.getArgument(0, regT1, regT0); // reload last result registers.
130     skipTimeout.link(this);
131 }
132 #elif USE(JSVALUE32_64)
133 void JIT::emitTimeoutCheck()
134 {
135     Jump skipTimeout = branchSub32(NonZero, TrustedImm32(1), timeoutCheckRegister);
136     JITStubCall stubCall(this, cti_timeout_check);
137     stubCall.addArgument(regT1, regT0); // save last result registers.
138     stubCall.call(timeoutCheckRegister);
139     stubCall.getArgument(0, regT1, regT0); // reload last result registers.
140     skipTimeout.link(this);
141 }
142 #else
143 void JIT::emitTimeoutCheck()
144 {
145     Jump skipTimeout = branchSub32(NonZero, TrustedImm32(1), timeoutCheckRegister);
146     JITStubCall(this, cti_timeout_check).call(timeoutCheckRegister);
147     skipTimeout.link(this);
148
149     killLastResultRegister();
150 }
151 #endif
152
153 #define NEXT_OPCODE(name) \
154     m_bytecodeOffset += OPCODE_LENGTH(name); \
155     break;
156
157 #if USE(JSVALUE32_64)
158 #define DEFINE_BINARY_OP(name) \
159     case name: { \
160         JITStubCall stubCall(this, cti_##name); \
161         stubCall.addArgument(currentInstruction[2].u.operand); \
162         stubCall.addArgument(currentInstruction[3].u.operand); \
163         stubCall.call(currentInstruction[1].u.operand); \
164         NEXT_OPCODE(name); \
165     }
166
167 #define DEFINE_UNARY_OP(name) \
168     case name: { \
169         JITStubCall stubCall(this, cti_##name); \
170         stubCall.addArgument(currentInstruction[2].u.operand); \
171         stubCall.call(currentInstruction[1].u.operand); \
172         NEXT_OPCODE(name); \
173     }
174
175 #else // USE(JSVALUE32_64)
176
177 #define DEFINE_BINARY_OP(name) \
178     case name: { \
179         JITStubCall stubCall(this, cti_##name); \
180         stubCall.addArgument(currentInstruction[2].u.operand, regT2); \
181         stubCall.addArgument(currentInstruction[3].u.operand, regT2); \
182         stubCall.call(currentInstruction[1].u.operand); \
183         NEXT_OPCODE(name); \
184     }
185
186 #define DEFINE_UNARY_OP(name) \
187     case name: { \
188         JITStubCall stubCall(this, cti_##name); \
189         stubCall.addArgument(currentInstruction[2].u.operand, regT2); \
190         stubCall.call(currentInstruction[1].u.operand); \
191         NEXT_OPCODE(name); \
192     }
193 #endif // USE(JSVALUE32_64)
194
195 #define DEFINE_OP(name) \
196     case name: { \
197         emit_##name(currentInstruction); \
198         NEXT_OPCODE(name); \
199     }
200
201 #define DEFINE_SLOWCASE_OP(name) \
202     case name: { \
203         emitSlow_##name(currentInstruction, iter); \
204         NEXT_OPCODE(name); \
205     }
206
207 void JIT::privateCompileMainPass()
208 {
209     Instruction* instructionsBegin = m_codeBlock->instructions().begin();
210     unsigned instructionCount = m_codeBlock->instructions().size();
211
212     m_globalResolveInfoIndex = 0;
213     m_callLinkInfoIndex = 0;
214
215     for (m_bytecodeOffset = 0; m_bytecodeOffset < instructionCount; ) {
216         if (m_disassembler)
217             m_disassembler->setForBytecodeMainPath(m_bytecodeOffset, label());
218         Instruction* currentInstruction = instructionsBegin + m_bytecodeOffset;
219         ASSERT_WITH_MESSAGE(m_interpreter->isOpcode(currentInstruction->u.opcode), "privateCompileMainPass gone bad @ %d", m_bytecodeOffset);
220
221 #if ENABLE(OPCODE_SAMPLING)
222         if (m_bytecodeOffset > 0) // Avoid the overhead of sampling op_enter twice.
223             sampleInstruction(currentInstruction);
224 #endif
225
226 #if USE(JSVALUE64)
227         if (atJumpTarget())
228             killLastResultRegister();
229 #endif
230
231         m_labels[m_bytecodeOffset] = label();
232
233 #if ENABLE(JIT_VERBOSE)
234         dataLogF("Old JIT emitting code for bc#%u at offset 0x%lx.\n", m_bytecodeOffset, (long)debugOffset());
235 #endif
236         
237         OpcodeID opcodeID = m_interpreter->getOpcodeID(currentInstruction->u.opcode);
238
239         if (m_compilation && opcodeID != op_call_put_result) {
240             add64(
241                 TrustedImm32(1),
242                 AbsoluteAddress(m_compilation->executionCounterFor(Profiler::OriginStack(Profiler::Origin(
243                     m_compilation->bytecodes(), m_bytecodeOffset)))->address()));
244         }
245
246         switch (opcodeID) {
247         DEFINE_BINARY_OP(op_del_by_val)
248         DEFINE_BINARY_OP(op_in)
249         DEFINE_BINARY_OP(op_less)
250         DEFINE_BINARY_OP(op_lesseq)
251         DEFINE_BINARY_OP(op_greater)
252         DEFINE_BINARY_OP(op_greatereq)
253         DEFINE_UNARY_OP(op_is_function)
254         DEFINE_UNARY_OP(op_is_object)
255         DEFINE_UNARY_OP(op_typeof)
256
257         DEFINE_OP(op_add)
258         DEFINE_OP(op_bitand)
259         DEFINE_OP(op_bitor)
260         DEFINE_OP(op_bitxor)
261         DEFINE_OP(op_call)
262         DEFINE_OP(op_call_eval)
263         DEFINE_OP(op_call_varargs)
264         DEFINE_OP(op_catch)
265         DEFINE_OP(op_construct)
266         DEFINE_OP(op_get_callee)
267         DEFINE_OP(op_create_this)
268         DEFINE_OP(op_convert_this)
269         DEFINE_OP(op_init_lazy_reg)
270         DEFINE_OP(op_create_arguments)
271         DEFINE_OP(op_debug)
272         DEFINE_OP(op_del_by_id)
273         DEFINE_OP(op_div)
274         DEFINE_OP(op_end)
275         DEFINE_OP(op_enter)
276         DEFINE_OP(op_create_activation)
277         DEFINE_OP(op_eq)
278         DEFINE_OP(op_eq_null)
279         case op_get_by_id_out_of_line:
280         case op_get_array_length:
281         DEFINE_OP(op_get_by_id)
282         DEFINE_OP(op_get_arguments_length)
283         DEFINE_OP(op_get_by_val)
284         DEFINE_OP(op_get_argument_by_val)
285         DEFINE_OP(op_get_by_pname)
286         DEFINE_OP(op_get_pnames)
287         DEFINE_OP(op_check_has_instance)
288         DEFINE_OP(op_instanceof)
289         DEFINE_OP(op_is_undefined)
290         DEFINE_OP(op_is_boolean)
291         DEFINE_OP(op_is_number)
292         DEFINE_OP(op_is_string)
293         DEFINE_OP(op_jeq_null)
294         DEFINE_OP(op_jfalse)
295         DEFINE_OP(op_jmp)
296         DEFINE_OP(op_jmp_scopes)
297         DEFINE_OP(op_jneq_null)
298         DEFINE_OP(op_jneq_ptr)
299         DEFINE_OP(op_jless)
300         DEFINE_OP(op_jlesseq)
301         DEFINE_OP(op_jgreater)
302         DEFINE_OP(op_jgreatereq)
303         DEFINE_OP(op_jnless)
304         DEFINE_OP(op_jnlesseq)
305         DEFINE_OP(op_jngreater)
306         DEFINE_OP(op_jngreatereq)
307         DEFINE_OP(op_jtrue)
308         DEFINE_OP(op_loop)
309         DEFINE_OP(op_loop_hint)
310         DEFINE_OP(op_loop_if_less)
311         DEFINE_OP(op_loop_if_lesseq)
312         DEFINE_OP(op_loop_if_greater)
313         DEFINE_OP(op_loop_if_greatereq)
314         DEFINE_OP(op_loop_if_true)
315         DEFINE_OP(op_loop_if_false)
316         DEFINE_OP(op_lshift)
317         DEFINE_OP(op_mod)
318         DEFINE_OP(op_mov)
319         DEFINE_OP(op_mul)
320         DEFINE_OP(op_negate)
321         DEFINE_OP(op_neq)
322         DEFINE_OP(op_neq_null)
323         DEFINE_OP(op_new_array)
324         DEFINE_OP(op_new_array_with_size)
325         DEFINE_OP(op_new_array_buffer)
326         DEFINE_OP(op_new_func)
327         DEFINE_OP(op_new_func_exp)
328         DEFINE_OP(op_new_object)
329         DEFINE_OP(op_new_regexp)
330         DEFINE_OP(op_next_pname)
331         DEFINE_OP(op_not)
332         DEFINE_OP(op_nstricteq)
333         DEFINE_OP(op_pop_scope)
334         DEFINE_OP(op_post_dec)
335         DEFINE_OP(op_post_inc)
336         DEFINE_OP(op_pre_dec)
337         DEFINE_OP(op_pre_inc)
338         DEFINE_OP(op_profile_did_call)
339         DEFINE_OP(op_profile_will_call)
340         DEFINE_OP(op_push_name_scope)
341         DEFINE_OP(op_push_with_scope)
342         case op_put_by_id_out_of_line:
343         case op_put_by_id_transition_direct:
344         case op_put_by_id_transition_normal:
345         case op_put_by_id_transition_direct_out_of_line:
346         case op_put_by_id_transition_normal_out_of_line:
347         DEFINE_OP(op_put_by_id)
348         DEFINE_OP(op_put_by_index)
349         DEFINE_OP(op_put_by_val)
350         DEFINE_OP(op_put_getter_setter)
351         case op_init_global_const_nop:
352             NEXT_OPCODE(op_init_global_const_nop);
353         DEFINE_OP(op_init_global_const)
354         DEFINE_OP(op_init_global_const_check)
355
356         case op_resolve_global_property:
357         case op_resolve_global_var:
358         case op_resolve_scoped_var:
359         case op_resolve_scoped_var_on_top_scope:
360         case op_resolve_scoped_var_with_top_scope_check:
361         DEFINE_OP(op_resolve)
362
363         case op_resolve_base_to_global:
364         case op_resolve_base_to_global_dynamic:
365         case op_resolve_base_to_scope:
366         case op_resolve_base_to_scope_with_top_scope_check:
367         DEFINE_OP(op_resolve_base)
368
369         case op_put_to_base_variable:
370         DEFINE_OP(op_put_to_base)
371
372         DEFINE_OP(op_ensure_property_exists)
373         DEFINE_OP(op_resolve_with_base)
374         DEFINE_OP(op_resolve_with_this)
375         DEFINE_OP(op_ret)
376         DEFINE_OP(op_call_put_result)
377         DEFINE_OP(op_ret_object_or_this)
378         DEFINE_OP(op_rshift)
379         DEFINE_OP(op_urshift)
380         DEFINE_OP(op_strcat)
381         DEFINE_OP(op_stricteq)
382         DEFINE_OP(op_sub)
383         DEFINE_OP(op_switch_char)
384         DEFINE_OP(op_switch_imm)
385         DEFINE_OP(op_switch_string)
386         DEFINE_OP(op_tear_off_activation)
387         DEFINE_OP(op_tear_off_arguments)
388         DEFINE_OP(op_throw)
389         DEFINE_OP(op_throw_static_error)
390         DEFINE_OP(op_to_jsnumber)
391         DEFINE_OP(op_to_primitive)
392
393         case op_get_by_id_chain:
394         case op_get_by_id_generic:
395         case op_get_by_id_proto:
396         case op_get_by_id_self:
397         case op_get_by_id_getter_chain:
398         case op_get_by_id_getter_proto:
399         case op_get_by_id_getter_self:
400         case op_get_by_id_custom_chain:
401         case op_get_by_id_custom_proto:
402         case op_get_by_id_custom_self:
403         case op_get_string_length:
404         case op_put_by_id_generic:
405         case op_put_by_id_replace:
406         case op_put_by_id_transition:
407             RELEASE_ASSERT_NOT_REACHED();
408         }
409     }
410
411     RELEASE_ASSERT(m_callLinkInfoIndex == m_callStructureStubCompilationInfo.size());
412
413 #ifndef NDEBUG
414     // Reset this, in order to guard its use with ASSERTs.
415     m_bytecodeOffset = (unsigned)-1;
416 #endif
417 }
418
419 void JIT::privateCompileLinkPass()
420 {
421     unsigned jmpTableCount = m_jmpTable.size();
422     for (unsigned i = 0; i < jmpTableCount; ++i)
423         m_jmpTable[i].from.linkTo(m_labels[m_jmpTable[i].toBytecodeOffset], this);
424     m_jmpTable.clear();
425 }
426
427 void JIT::privateCompileSlowCases()
428 {
429     Instruction* instructionsBegin = m_codeBlock->instructions().begin();
430
431     m_propertyAccessInstructionIndex = 0;
432     m_byValInstructionIndex = 0;
433     m_globalResolveInfoIndex = 0;
434     m_callLinkInfoIndex = 0;
435     
436 #if ENABLE(VALUE_PROFILER)
437     // Use this to assert that slow-path code associates new profiling sites with existing
438     // ValueProfiles rather than creating new ones. This ensures that for a given instruction
439     // (say, get_by_id) we get combined statistics for both the fast-path executions of that
440     // instructions and the slow-path executions. Furthermore, if the slow-path code created
441     // new ValueProfiles then the ValueProfiles would no longer be sorted by bytecode offset,
442     // which would break the invariant necessary to use CodeBlock::valueProfileForBytecodeOffset().
443     unsigned numberOfValueProfiles = m_codeBlock->numberOfValueProfiles();
444 #endif
445
446     for (Vector<SlowCaseEntry>::iterator iter = m_slowCases.begin(); iter != m_slowCases.end();) {
447 #if USE(JSVALUE64)
448         killLastResultRegister();
449 #endif
450
451         m_bytecodeOffset = iter->to;
452
453         unsigned firstTo = m_bytecodeOffset;
454
455         Instruction* currentInstruction = instructionsBegin + m_bytecodeOffset;
456         
457 #if ENABLE(VALUE_PROFILER)
458         RareCaseProfile* rareCaseProfile = 0;
459         if (shouldEmitProfiling())
460             rareCaseProfile = m_codeBlock->addRareCaseProfile(m_bytecodeOffset);
461 #endif
462
463 #if ENABLE(JIT_VERBOSE)
464         dataLogF("Old JIT emitting slow code for bc#%u at offset 0x%lx.\n", m_bytecodeOffset, (long)debugOffset());
465 #endif
466         
467         if (m_disassembler)
468             m_disassembler->setForBytecodeSlowPath(m_bytecodeOffset, label());
469
470         switch (m_interpreter->getOpcodeID(currentInstruction->u.opcode)) {
471         DEFINE_SLOWCASE_OP(op_add)
472         DEFINE_SLOWCASE_OP(op_bitand)
473         DEFINE_SLOWCASE_OP(op_bitor)
474         DEFINE_SLOWCASE_OP(op_bitxor)
475         DEFINE_SLOWCASE_OP(op_call)
476         DEFINE_SLOWCASE_OP(op_call_eval)
477         DEFINE_SLOWCASE_OP(op_call_varargs)
478         DEFINE_SLOWCASE_OP(op_construct)
479         DEFINE_SLOWCASE_OP(op_convert_this)
480         DEFINE_SLOWCASE_OP(op_create_this)
481         DEFINE_SLOWCASE_OP(op_div)
482         DEFINE_SLOWCASE_OP(op_eq)
483         case op_get_by_id_out_of_line:
484         case op_get_array_length:
485         DEFINE_SLOWCASE_OP(op_get_by_id)
486         DEFINE_SLOWCASE_OP(op_get_arguments_length)
487         DEFINE_SLOWCASE_OP(op_get_by_val)
488         DEFINE_SLOWCASE_OP(op_get_argument_by_val)
489         DEFINE_SLOWCASE_OP(op_get_by_pname)
490         DEFINE_SLOWCASE_OP(op_check_has_instance)
491         DEFINE_SLOWCASE_OP(op_instanceof)
492         DEFINE_SLOWCASE_OP(op_jfalse)
493         DEFINE_SLOWCASE_OP(op_jless)
494         DEFINE_SLOWCASE_OP(op_jlesseq)
495         DEFINE_SLOWCASE_OP(op_jgreater)
496         DEFINE_SLOWCASE_OP(op_jgreatereq)
497         DEFINE_SLOWCASE_OP(op_jnless)
498         DEFINE_SLOWCASE_OP(op_jnlesseq)
499         DEFINE_SLOWCASE_OP(op_jngreater)
500         DEFINE_SLOWCASE_OP(op_jngreatereq)
501         DEFINE_SLOWCASE_OP(op_jtrue)
502         DEFINE_SLOWCASE_OP(op_loop_if_less)
503         DEFINE_SLOWCASE_OP(op_loop_if_lesseq)
504         DEFINE_SLOWCASE_OP(op_loop_if_greater)
505         DEFINE_SLOWCASE_OP(op_loop_if_greatereq)
506         DEFINE_SLOWCASE_OP(op_loop_if_true)
507         DEFINE_SLOWCASE_OP(op_loop_if_false)
508         DEFINE_SLOWCASE_OP(op_lshift)
509         DEFINE_SLOWCASE_OP(op_mod)
510         DEFINE_SLOWCASE_OP(op_mul)
511         DEFINE_SLOWCASE_OP(op_negate)
512         DEFINE_SLOWCASE_OP(op_neq)
513         DEFINE_SLOWCASE_OP(op_new_object)
514         DEFINE_SLOWCASE_OP(op_not)
515         DEFINE_SLOWCASE_OP(op_nstricteq)
516         DEFINE_SLOWCASE_OP(op_post_dec)
517         DEFINE_SLOWCASE_OP(op_post_inc)
518         DEFINE_SLOWCASE_OP(op_pre_dec)
519         DEFINE_SLOWCASE_OP(op_pre_inc)
520         case op_put_by_id_out_of_line:
521         case op_put_by_id_transition_direct:
522         case op_put_by_id_transition_normal:
523         case op_put_by_id_transition_direct_out_of_line:
524         case op_put_by_id_transition_normal_out_of_line:
525         DEFINE_SLOWCASE_OP(op_put_by_id)
526         DEFINE_SLOWCASE_OP(op_put_by_val)
527         DEFINE_SLOWCASE_OP(op_init_global_const_check);
528         DEFINE_SLOWCASE_OP(op_rshift)
529         DEFINE_SLOWCASE_OP(op_urshift)
530         DEFINE_SLOWCASE_OP(op_stricteq)
531         DEFINE_SLOWCASE_OP(op_sub)
532         DEFINE_SLOWCASE_OP(op_to_jsnumber)
533         DEFINE_SLOWCASE_OP(op_to_primitive)
534
535         case op_resolve_global_property:
536         case op_resolve_global_var:
537         case op_resolve_scoped_var:
538         case op_resolve_scoped_var_on_top_scope:
539         case op_resolve_scoped_var_with_top_scope_check:
540         DEFINE_SLOWCASE_OP(op_resolve)
541
542         case op_resolve_base_to_global:
543         case op_resolve_base_to_global_dynamic:
544         case op_resolve_base_to_scope:
545         case op_resolve_base_to_scope_with_top_scope_check:
546         DEFINE_SLOWCASE_OP(op_resolve_base)
547         DEFINE_SLOWCASE_OP(op_resolve_with_base)
548         DEFINE_SLOWCASE_OP(op_resolve_with_this)
549
550         case op_put_to_base_variable:
551         DEFINE_SLOWCASE_OP(op_put_to_base)
552
553         default:
554             RELEASE_ASSERT_NOT_REACHED();
555         }
556
557         RELEASE_ASSERT_WITH_MESSAGE(iter == m_slowCases.end() || firstTo != iter->to, "Not enough jumps linked in slow case codegen.");
558         RELEASE_ASSERT_WITH_MESSAGE(firstTo == (iter - 1)->to, "Too many jumps linked in slow case codegen.");
559         
560 #if ENABLE(VALUE_PROFILER)
561         if (shouldEmitProfiling())
562             add32(TrustedImm32(1), AbsoluteAddress(&rareCaseProfile->m_counter));
563 #endif
564
565         emitJumpSlowToHot(jump(), 0);
566     }
567
568     RELEASE_ASSERT(m_propertyAccessInstructionIndex == m_propertyAccessCompilationInfo.size());
569     RELEASE_ASSERT(m_callLinkInfoIndex == m_callStructureStubCompilationInfo.size());
570 #if ENABLE(VALUE_PROFILER)
571     RELEASE_ASSERT(numberOfValueProfiles == m_codeBlock->numberOfValueProfiles());
572 #endif
573
574 #ifndef NDEBUG
575     // Reset this, in order to guard its use with ASSERTs.
576     m_bytecodeOffset = (unsigned)-1;
577 #endif
578 }
579
580 ALWAYS_INLINE void PropertyStubCompilationInfo::copyToStubInfo(StructureStubInfo& info, LinkBuffer &linkBuffer)
581 {
582     ASSERT(bytecodeIndex != std::numeric_limits<unsigned>::max());
583     info.bytecodeIndex = bytecodeIndex;
584     info.callReturnLocation = linkBuffer.locationOf(callReturnLocation);
585     info.hotPathBegin = linkBuffer.locationOf(hotPathBegin);
586
587     switch (m_type) {
588     case GetById: {
589         CodeLocationLabel hotPathBeginLocation = linkBuffer.locationOf(hotPathBegin);
590         info.patch.baseline.u.get.structureToCompare = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getStructureToCompare));
591         info.patch.baseline.u.get.structureCheck = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getStructureCheck));
592         info.patch.baseline.u.get.propertyStorageLoad = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(propertyStorageLoad));
593 #if USE(JSVALUE64)
594         info.patch.baseline.u.get.displacementLabel = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getDisplacementLabel));
595 #else
596         info.patch.baseline.u.get.displacementLabel1 = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getDisplacementLabel1));
597         info.patch.baseline.u.get.displacementLabel2 = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getDisplacementLabel2));
598 #endif
599         info.patch.baseline.u.get.putResult = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(getPutResult));
600         info.patch.baseline.u.get.coldPathBegin = MacroAssembler::differenceBetweenCodePtr(linkBuffer.locationOf(getColdPathBegin), linkBuffer.locationOf(callReturnLocation));
601         break;
602     }
603     case PutById:
604         CodeLocationLabel hotPathBeginLocation = linkBuffer.locationOf(hotPathBegin);
605         info.patch.baseline.u.put.structureToCompare = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(putStructureToCompare));
606         info.patch.baseline.u.put.propertyStorageLoad = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(propertyStorageLoad));
607 #if USE(JSVALUE64)
608         info.patch.baseline.u.put.displacementLabel = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(putDisplacementLabel));
609 #else
610         info.patch.baseline.u.put.displacementLabel1 = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(putDisplacementLabel1));
611         info.patch.baseline.u.put.displacementLabel2 = MacroAssembler::differenceBetweenCodePtr(hotPathBeginLocation, linkBuffer.locationOf(putDisplacementLabel2));
612 #endif
613         break;
614     }
615 }
616
617 JITCode JIT::privateCompile(CodePtr* functionEntryArityCheck, JITCompilationEffort effort)
618 {
619 #if ENABLE(JIT_VERBOSE_OSR)
620     printf("Compiling JIT code!\n");
621 #endif
622     
623 #if ENABLE(VALUE_PROFILER)
624     DFG::CapabilityLevel level = m_codeBlock->canCompileWithDFG();
625     switch (level) {
626     case DFG::CannotCompile:
627         m_canBeOptimized = false;
628         m_shouldEmitProfiling = false;
629         break;
630     case DFG::ShouldProfile:
631         m_canBeOptimized = false;
632         m_shouldEmitProfiling = true;
633         break;
634     case DFG::CanCompile:
635         m_canBeOptimized = true;
636         m_shouldEmitProfiling = true;
637         break;
638     default:
639         RELEASE_ASSERT_NOT_REACHED();
640         break;
641     }
642 #endif
643     
644     if (Options::showDisassembly() || m_globalData->m_perBytecodeProfiler)
645         m_disassembler = adoptPtr(new JITDisassembler(m_codeBlock));
646     if (m_globalData->m_perBytecodeProfiler) {
647         m_compilation = m_globalData->m_perBytecodeProfiler->newCompilation(m_codeBlock, Profiler::Baseline);
648         m_compilation->addProfiledBytecodes(*m_globalData->m_perBytecodeProfiler, m_codeBlock);
649     }
650     
651     if (m_disassembler)
652         m_disassembler->setStartOfCode(label());
653
654     // Just add a little bit of randomness to the codegen
655     if (m_randomGenerator.getUint32() & 1)
656         nop();
657
658     preserveReturnAddressAfterCall(regT2);
659     emitPutToCallFrameHeader(regT2, JSStack::ReturnPC);
660     emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
661
662     Label beginLabel(this);
663
664     sampleCodeBlock(m_codeBlock);
665 #if ENABLE(OPCODE_SAMPLING)
666     sampleInstruction(m_codeBlock->instructions().begin());
667 #endif
668
669     Jump stackCheck;
670     if (m_codeBlock->codeType() == FunctionCode) {
671 #if ENABLE(DFG_JIT)
672 #if DFG_ENABLE(SUCCESS_STATS)
673         static SamplingCounter counter("orignalJIT");
674         emitCount(counter);
675 #endif
676 #endif
677
678 #if ENABLE(VALUE_PROFILER)
679         ASSERT(m_bytecodeOffset == (unsigned)-1);
680         if (shouldEmitProfiling()) {
681             for (int argument = 0; argument < m_codeBlock->numParameters(); ++argument) {
682                 // If this is a constructor, then we want to put in a dummy profiling site (to
683                 // keep things consistent) but we don't actually want to record the dummy value.
684                 if (m_codeBlock->m_isConstructor && !argument)
685                     continue;
686                 int offset = CallFrame::argumentOffsetIncludingThis(argument) * static_cast<int>(sizeof(Register));
687 #if USE(JSVALUE64)
688                 load64(Address(callFrameRegister, offset), regT0);
689 #elif USE(JSVALUE32_64)
690                 load32(Address(callFrameRegister, offset + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), regT0);
691                 load32(Address(callFrameRegister, offset + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), regT1);
692 #endif
693                 emitValueProfilingSite(m_codeBlock->valueProfileForArgument(argument));
694             }
695         }
696 #endif
697
698         addPtr(TrustedImm32(m_codeBlock->m_numCalleeRegisters * sizeof(Register)), callFrameRegister, regT1);
699         stackCheck = branchPtr(Below, AbsoluteAddress(m_globalData->interpreter->stack().addressOfEnd()), regT1);
700     }
701
702     Label functionBody = label();
703     
704     privateCompileMainPass();
705     privateCompileLinkPass();
706     privateCompileSlowCases();
707     
708     if (m_disassembler)
709         m_disassembler->setEndOfSlowPath(label());
710
711     Label arityCheck;
712     if (m_codeBlock->codeType() == FunctionCode) {
713         stackCheck.link(this);
714         m_bytecodeOffset = 0;
715         JITStubCall(this, cti_stack_check).call();
716 #ifndef NDEBUG
717         m_bytecodeOffset = (unsigned)-1; // Reset this, in order to guard its use with ASSERTs.
718 #endif
719         jump(functionBody);
720
721         arityCheck = label();
722         preserveReturnAddressAfterCall(regT2);
723         emitPutToCallFrameHeader(regT2, JSStack::ReturnPC);
724         emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
725
726         load32(payloadFor(JSStack::ArgumentCount), regT1);
727         branch32(AboveOrEqual, regT1, TrustedImm32(m_codeBlock->m_numParameters)).linkTo(beginLabel, this);
728
729         m_bytecodeOffset = 0;
730         JITStubCall(this, m_codeBlock->m_isConstructor ? cti_op_construct_arityCheck : cti_op_call_arityCheck).call(callFrameRegister);
731 #if !ASSERT_DISABLED
732         m_bytecodeOffset = (unsigned)-1; // Reset this, in order to guard its use with ASSERTs.
733 #endif
734
735         jump(beginLabel);
736     }
737
738     ASSERT(m_jmpTable.isEmpty());
739     
740     if (m_disassembler)
741         m_disassembler->setEndOfCode(label());
742
743     LinkBuffer patchBuffer(*m_globalData, this, m_codeBlock, effort);
744     if (patchBuffer.didFailToAllocate())
745         return JITCode();
746
747     // Translate vPC offsets into addresses in JIT generated code, for switch tables.
748     for (unsigned i = 0; i < m_switches.size(); ++i) {
749         SwitchRecord record = m_switches[i];
750         unsigned bytecodeOffset = record.bytecodeOffset;
751
752         if (record.type != SwitchRecord::String) {
753             ASSERT(record.type == SwitchRecord::Immediate || record.type == SwitchRecord::Character); 
754             ASSERT(record.jumpTable.simpleJumpTable->branchOffsets.size() == record.jumpTable.simpleJumpTable->ctiOffsets.size());
755
756             record.jumpTable.simpleJumpTable->ctiDefault = patchBuffer.locationOf(m_labels[bytecodeOffset + record.defaultOffset]);
757
758             for (unsigned j = 0; j < record.jumpTable.simpleJumpTable->branchOffsets.size(); ++j) {
759                 unsigned offset = record.jumpTable.simpleJumpTable->branchOffsets[j];
760                 record.jumpTable.simpleJumpTable->ctiOffsets[j] = offset ? patchBuffer.locationOf(m_labels[bytecodeOffset + offset]) : record.jumpTable.simpleJumpTable->ctiDefault;
761             }
762         } else {
763             ASSERT(record.type == SwitchRecord::String);
764
765             record.jumpTable.stringJumpTable->ctiDefault = patchBuffer.locationOf(m_labels[bytecodeOffset + record.defaultOffset]);
766
767             StringJumpTable::StringOffsetTable::iterator end = record.jumpTable.stringJumpTable->offsetTable.end();            
768             for (StringJumpTable::StringOffsetTable::iterator it = record.jumpTable.stringJumpTable->offsetTable.begin(); it != end; ++it) {
769                 unsigned offset = it->value.branchOffset;
770                 it->value.ctiOffset = offset ? patchBuffer.locationOf(m_labels[bytecodeOffset + offset]) : record.jumpTable.stringJumpTable->ctiDefault;
771             }
772         }
773     }
774
775     for (size_t i = 0; i < m_codeBlock->numberOfExceptionHandlers(); ++i) {
776         HandlerInfo& handler = m_codeBlock->exceptionHandler(i);
777         handler.nativeCode = patchBuffer.locationOf(m_labels[handler.target]);
778     }
779
780     for (Vector<CallRecord>::iterator iter = m_calls.begin(); iter != m_calls.end(); ++iter) {
781         if (iter->to)
782             patchBuffer.link(iter->from, FunctionPtr(iter->to));
783     }
784
785     m_codeBlock->callReturnIndexVector().reserveCapacity(m_calls.size());
786     for (Vector<CallRecord>::iterator iter = m_calls.begin(); iter != m_calls.end(); ++iter)
787         m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(patchBuffer.returnAddressOffset(iter->from), iter->bytecodeOffset));
788
789     m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccessCompilationInfo.size());
790     for (unsigned i = 0; i < m_propertyAccessCompilationInfo.size(); ++i)
791         m_propertyAccessCompilationInfo[i].copyToStubInfo(m_codeBlock->structureStubInfo(i), patchBuffer);
792     m_codeBlock->setNumberOfByValInfos(m_byValCompilationInfo.size());
793     for (unsigned i = 0; i < m_byValCompilationInfo.size(); ++i) {
794         CodeLocationJump badTypeJump = CodeLocationJump(patchBuffer.locationOf(m_byValCompilationInfo[i].badTypeJump));
795         CodeLocationLabel doneTarget = patchBuffer.locationOf(m_byValCompilationInfo[i].doneTarget);
796         CodeLocationLabel slowPathTarget = patchBuffer.locationOf(m_byValCompilationInfo[i].slowPathTarget);
797         CodeLocationCall returnAddress = patchBuffer.locationOf(m_byValCompilationInfo[i].returnAddress);
798         
799         m_codeBlock->byValInfo(i) = ByValInfo(
800             m_byValCompilationInfo[i].bytecodeIndex,
801             badTypeJump,
802             m_byValCompilationInfo[i].arrayMode,
803             differenceBetweenCodePtr(badTypeJump, doneTarget),
804             differenceBetweenCodePtr(returnAddress, slowPathTarget));
805     }
806     m_codeBlock->setNumberOfCallLinkInfos(m_callStructureStubCompilationInfo.size());
807     for (unsigned i = 0; i < m_codeBlock->numberOfCallLinkInfos(); ++i) {
808         CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
809         info.callType = m_callStructureStubCompilationInfo[i].callType;
810         info.codeOrigin = CodeOrigin(m_callStructureStubCompilationInfo[i].bytecodeIndex);
811         info.callReturnLocation = patchBuffer.locationOfNearCall(m_callStructureStubCompilationInfo[i].callReturnLocation);
812         info.hotPathBegin = patchBuffer.locationOf(m_callStructureStubCompilationInfo[i].hotPathBegin);
813         info.hotPathOther = patchBuffer.locationOfNearCall(m_callStructureStubCompilationInfo[i].hotPathOther);
814         info.calleeGPR = regT0;
815     }
816
817 #if ENABLE(DFG_JIT) || ENABLE(LLINT)
818     if (canBeOptimized()
819 #if ENABLE(LLINT)
820         || true
821 #endif
822         ) {
823         CompactJITCodeMap::Encoder jitCodeMapEncoder;
824         for (unsigned bytecodeOffset = 0; bytecodeOffset < m_labels.size(); ++bytecodeOffset) {
825             if (m_labels[bytecodeOffset].isSet())
826                 jitCodeMapEncoder.append(bytecodeOffset, patchBuffer.offsetOf(m_labels[bytecodeOffset]));
827         }
828         m_codeBlock->setJITCodeMap(jitCodeMapEncoder.finish());
829     }
830 #endif
831
832     if (m_codeBlock->codeType() == FunctionCode && functionEntryArityCheck)
833         *functionEntryArityCheck = patchBuffer.locationOf(arityCheck);
834
835     if (Options::showDisassembly())
836         m_disassembler->dump(patchBuffer);
837     if (m_compilation)
838         m_disassembler->reportToProfiler(m_compilation.get(), patchBuffer);
839     
840     CodeRef result = patchBuffer.finalizeCodeWithoutDisassembly();
841     
842     m_globalData->machineCodeBytesPerBytecodeWordForBaselineJIT.add(
843         static_cast<double>(result.size()) /
844         static_cast<double>(m_codeBlock->instructions().size()));
845     
846     m_codeBlock->shrinkToFit(CodeBlock::LateShrink);
847     
848 #if ENABLE(JIT_VERBOSE)
849     dataLogF("JIT generated code for %p at [%p, %p).\n", m_codeBlock, result.executableMemory()->start(), result.executableMemory()->end());
850 #endif
851     
852     return JITCode(result, JITCode::BaselineJIT);
853 }
854
855 void JIT::linkFor(JSFunction* callee, CodeBlock* callerCodeBlock, CodeBlock* calleeCodeBlock, JIT::CodePtr code, CallLinkInfo* callLinkInfo, JSGlobalData* globalData, CodeSpecializationKind kind)
856 {
857     RepatchBuffer repatchBuffer(callerCodeBlock);
858
859     ASSERT(!callLinkInfo->isLinked());
860     callLinkInfo->callee.set(*globalData, callLinkInfo->hotPathBegin, callerCodeBlock->ownerExecutable(), callee);
861     callLinkInfo->lastSeenCallee.set(*globalData, callerCodeBlock->ownerExecutable(), callee);
862     repatchBuffer.relink(callLinkInfo->hotPathOther, code);
863
864     if (calleeCodeBlock)
865         calleeCodeBlock->linkIncomingCall(callLinkInfo);
866
867     // Patch the slow patch so we do not continue to try to link.
868     if (kind == CodeForCall) {
869         ASSERT(callLinkInfo->callType == CallLinkInfo::Call
870                || callLinkInfo->callType == CallLinkInfo::CallVarargs);
871         if (callLinkInfo->callType == CallLinkInfo::Call) {
872             repatchBuffer.relink(callLinkInfo->callReturnLocation, globalData->getCTIStub(linkClosureCallGenerator).code());
873             return;
874         }
875
876         repatchBuffer.relink(callLinkInfo->callReturnLocation, globalData->getCTIStub(virtualCallGenerator).code());
877         return;
878     }
879
880     ASSERT(kind == CodeForConstruct);
881     repatchBuffer.relink(callLinkInfo->callReturnLocation, globalData->getCTIStub(virtualConstructGenerator).code());
882 }
883
884 void JIT::linkSlowCall(CodeBlock* callerCodeBlock, CallLinkInfo* callLinkInfo)
885 {
886     RepatchBuffer repatchBuffer(callerCodeBlock);
887
888     repatchBuffer.relink(callLinkInfo->callReturnLocation, callerCodeBlock->globalData()->getCTIStub(virtualCallGenerator).code());
889 }
890
891 } // namespace JSC
892
893 #endif // ENABLE(JIT)