CallLinkInfo inside StructureStubInfo should not use polymorphic stubs
[WebKit-https.git] / Source / JavaScriptCore / jit / JITArithmetic.cpp
1 /*
2  * Copyright (C) 2008 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27
28 #if ENABLE(JIT)
29 #include "JIT.h"
30
31 #include "CodeBlock.h"
32 #include "JITInlines.h"
33 #include "JITOperations.h"
34 #include "JITStubs.h"
35 #include "JSArray.h"
36 #include "JSFunction.h"
37 #include "Interpreter.h"
38 #include "JSCInlines.h"
39 #include "ResultType.h"
40 #include "SamplingTool.h"
41 #include "SlowPathCall.h"
42
43
44 namespace JSC {
45
46 void JIT::emit_op_jless(Instruction* currentInstruction)
47 {
48     int op1 = currentInstruction[1].u.operand;
49     int op2 = currentInstruction[2].u.operand;
50     unsigned target = currentInstruction[3].u.operand;
51
52     emit_compareAndJump(op_jless, op1, op2, target, LessThan);
53 }
54
55 void JIT::emit_op_jlesseq(Instruction* currentInstruction)
56 {
57     int op1 = currentInstruction[1].u.operand;
58     int op2 = currentInstruction[2].u.operand;
59     unsigned target = currentInstruction[3].u.operand;
60
61     emit_compareAndJump(op_jlesseq, op1, op2, target, LessThanOrEqual);
62 }
63
64 void JIT::emit_op_jgreater(Instruction* currentInstruction)
65 {
66     int op1 = currentInstruction[1].u.operand;
67     int op2 = currentInstruction[2].u.operand;
68     unsigned target = currentInstruction[3].u.operand;
69
70     emit_compareAndJump(op_jgreater, op1, op2, target, GreaterThan);
71 }
72
73 void JIT::emit_op_jgreatereq(Instruction* currentInstruction)
74 {
75     int op1 = currentInstruction[1].u.operand;
76     int op2 = currentInstruction[2].u.operand;
77     unsigned target = currentInstruction[3].u.operand;
78
79     emit_compareAndJump(op_jgreatereq, op1, op2, target, GreaterThanOrEqual);
80 }
81
82 void JIT::emit_op_jnless(Instruction* currentInstruction)
83 {
84     int op1 = currentInstruction[1].u.operand;
85     int op2 = currentInstruction[2].u.operand;
86     unsigned target = currentInstruction[3].u.operand;
87
88     emit_compareAndJump(op_jnless, op1, op2, target, GreaterThanOrEqual);
89 }
90
91 void JIT::emit_op_jnlesseq(Instruction* currentInstruction)
92 {
93     int op1 = currentInstruction[1].u.operand;
94     int op2 = currentInstruction[2].u.operand;
95     unsigned target = currentInstruction[3].u.operand;
96
97     emit_compareAndJump(op_jnlesseq, op1, op2, target, GreaterThan);
98 }
99
100 void JIT::emit_op_jngreater(Instruction* currentInstruction)
101 {
102     int op1 = currentInstruction[1].u.operand;
103     int op2 = currentInstruction[2].u.operand;
104     unsigned target = currentInstruction[3].u.operand;
105
106     emit_compareAndJump(op_jngreater, op1, op2, target, LessThanOrEqual);
107 }
108
109 void JIT::emit_op_jngreatereq(Instruction* currentInstruction)
110 {
111     int op1 = currentInstruction[1].u.operand;
112     int op2 = currentInstruction[2].u.operand;
113     unsigned target = currentInstruction[3].u.operand;
114
115     emit_compareAndJump(op_jngreatereq, op1, op2, target, LessThan);
116 }
117
118 void JIT::emitSlow_op_jless(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
119 {
120     int op1 = currentInstruction[1].u.operand;
121     int op2 = currentInstruction[2].u.operand;
122     unsigned target = currentInstruction[3].u.operand;
123
124     emit_compareAndJumpSlow(op1, op2, target, DoubleLessThan, operationCompareLess, false, iter);
125 }
126
127 void JIT::emitSlow_op_jlesseq(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
128 {
129     int op1 = currentInstruction[1].u.operand;
130     int op2 = currentInstruction[2].u.operand;
131     unsigned target = currentInstruction[3].u.operand;
132
133     emit_compareAndJumpSlow(op1, op2, target, DoubleLessThanOrEqual, operationCompareLessEq, false, iter);
134 }
135
136 void JIT::emitSlow_op_jgreater(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
137 {
138     int op1 = currentInstruction[1].u.operand;
139     int op2 = currentInstruction[2].u.operand;
140     unsigned target = currentInstruction[3].u.operand;
141
142     emit_compareAndJumpSlow(op1, op2, target, DoubleGreaterThan, operationCompareGreater, false, iter);
143 }
144
145 void JIT::emitSlow_op_jgreatereq(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
146 {
147     int op1 = currentInstruction[1].u.operand;
148     int op2 = currentInstruction[2].u.operand;
149     unsigned target = currentInstruction[3].u.operand;
150
151     emit_compareAndJumpSlow(op1, op2, target, DoubleGreaterThanOrEqual, operationCompareGreaterEq, false, iter);
152 }
153
154 void JIT::emitSlow_op_jnless(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
155 {
156     int op1 = currentInstruction[1].u.operand;
157     int op2 = currentInstruction[2].u.operand;
158     unsigned target = currentInstruction[3].u.operand;
159
160     emit_compareAndJumpSlow(op1, op2, target, DoubleGreaterThanOrEqualOrUnordered, operationCompareLess, true, iter);
161 }
162
163 void JIT::emitSlow_op_jnlesseq(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
164 {
165     int op1 = currentInstruction[1].u.operand;
166     int op2 = currentInstruction[2].u.operand;
167     unsigned target = currentInstruction[3].u.operand;
168
169     emit_compareAndJumpSlow(op1, op2, target, DoubleGreaterThanOrUnordered, operationCompareLessEq, true, iter);
170 }
171
172 void JIT::emitSlow_op_jngreater(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
173 {
174     int op1 = currentInstruction[1].u.operand;
175     int op2 = currentInstruction[2].u.operand;
176     unsigned target = currentInstruction[3].u.operand;
177
178     emit_compareAndJumpSlow(op1, op2, target, DoubleLessThanOrEqualOrUnordered, operationCompareGreater, true, iter);
179 }
180
181 void JIT::emitSlow_op_jngreatereq(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
182 {
183     int op1 = currentInstruction[1].u.operand;
184     int op2 = currentInstruction[2].u.operand;
185     unsigned target = currentInstruction[3].u.operand;
186
187     emit_compareAndJumpSlow(op1, op2, target, DoubleLessThanOrUnordered, operationCompareGreaterEq, true, iter);
188 }
189
190 #if USE(JSVALUE64)
191
192 void JIT::emit_op_negate(Instruction* currentInstruction)
193 {
194     int dst = currentInstruction[1].u.operand;
195     int src = currentInstruction[2].u.operand;
196
197     emitGetVirtualRegister(src, regT0);
198
199     Jump srcNotInt = emitJumpIfNotImmediateInteger(regT0);
200     addSlowCase(branchTest32(Zero, regT0, TrustedImm32(0x7fffffff)));
201     neg32(regT0);
202     emitFastArithReTagImmediate(regT0, regT0);
203
204     Jump end = jump();
205
206     srcNotInt.link(this);
207     emitJumpSlowCaseIfNotImmediateNumber(regT0);
208
209     move(TrustedImm64((int64_t)0x8000000000000000ull), regT1);
210     xor64(regT1, regT0);
211
212     end.link(this);
213     emitPutVirtualRegister(dst);
214 }
215
216 void JIT::emitSlow_op_negate(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
217 {
218     linkSlowCase(iter); // 0x7fffffff check
219     linkSlowCase(iter); // double check
220
221     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_negate);
222     slowPathCall.call();
223 }
224
225 void JIT::emit_op_lshift(Instruction* currentInstruction)
226 {
227     int result = currentInstruction[1].u.operand;
228     int op1 = currentInstruction[2].u.operand;
229     int op2 = currentInstruction[3].u.operand;
230
231     emitGetVirtualRegisters(op1, regT0, op2, regT2);
232     // FIXME: would we be better using 'emitJumpSlowCaseIfNotImmediateIntegers'? - we *probably* ought to be consistent.
233     emitJumpSlowCaseIfNotImmediateInteger(regT0);
234     emitJumpSlowCaseIfNotImmediateInteger(regT2);
235     emitFastArithImmToInt(regT0);
236     emitFastArithImmToInt(regT2);
237     lshift32(regT2, regT0);
238     emitFastArithReTagImmediate(regT0, regT0);
239     emitPutVirtualRegister(result);
240 }
241
242 void JIT::emitSlow_op_lshift(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
243 {
244     linkSlowCase(iter);
245     linkSlowCase(iter);
246     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_lshift);
247     slowPathCall.call();
248 }
249
250 void JIT::emit_op_rshift(Instruction* currentInstruction)
251 {
252     int result = currentInstruction[1].u.operand;
253     int op1 = currentInstruction[2].u.operand;
254     int op2 = currentInstruction[3].u.operand;
255
256     if (isOperandConstantImmediateInt(op2)) {
257         // isOperandConstantImmediateInt(op2) => 1 SlowCase
258         emitGetVirtualRegister(op1, regT0);
259         emitJumpSlowCaseIfNotImmediateInteger(regT0);
260         // Mask with 0x1f as per ecma-262 11.7.2 step 7.
261         rshift32(Imm32(getConstantOperandImmediateInt(op2) & 0x1f), regT0);
262     } else {
263         emitGetVirtualRegisters(op1, regT0, op2, regT2);
264         if (supportsFloatingPointTruncate()) {
265             Jump lhsIsInt = emitJumpIfImmediateInteger(regT0);
266             // supportsFloatingPoint() && USE(JSVALUE64) => 3 SlowCases
267             addSlowCase(emitJumpIfNotImmediateNumber(regT0));
268             add64(tagTypeNumberRegister, regT0);
269             move64ToDouble(regT0, fpRegT0);
270             addSlowCase(branchTruncateDoubleToInt32(fpRegT0, regT0));
271             lhsIsInt.link(this);
272             emitJumpSlowCaseIfNotImmediateInteger(regT2);
273         } else {
274             // !supportsFloatingPoint() => 2 SlowCases
275             emitJumpSlowCaseIfNotImmediateInteger(regT0);
276             emitJumpSlowCaseIfNotImmediateInteger(regT2);
277         }
278         emitFastArithImmToInt(regT2);
279         rshift32(regT2, regT0);
280     }
281     emitFastArithIntToImmNoCheck(regT0, regT0);
282     emitPutVirtualRegister(result);
283 }
284
285 void JIT::emitSlow_op_rshift(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
286 {
287     int op2 = currentInstruction[3].u.operand;
288
289     if (isOperandConstantImmediateInt(op2))
290         linkSlowCase(iter);
291
292     else {
293         if (supportsFloatingPointTruncate()) {
294             linkSlowCase(iter);
295             linkSlowCase(iter);
296             linkSlowCase(iter);
297         } else {
298             linkSlowCase(iter);
299             linkSlowCase(iter);
300         }
301     }
302
303     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_rshift);
304     slowPathCall.call();
305 }
306
307 void JIT::emit_op_urshift(Instruction* currentInstruction)
308 {
309     int result = currentInstruction[1].u.operand;
310     int op1 = currentInstruction[2].u.operand;
311     int op2 = currentInstruction[3].u.operand;
312
313     if (isOperandConstantImmediateInt(op2)) {
314         // isOperandConstantImmediateInt(op2) => 1 SlowCase
315         emitGetVirtualRegister(op1, regT0);
316         emitJumpSlowCaseIfNotImmediateInteger(regT0);
317         // Mask with 0x1f as per ecma-262 11.7.2 step 7.
318         urshift32(Imm32(getConstantOperandImmediateInt(op2) & 0x1f), regT0);
319     } else {
320         emitGetVirtualRegisters(op1, regT0, op2, regT2);
321         if (supportsFloatingPointTruncate()) {
322             Jump lhsIsInt = emitJumpIfImmediateInteger(regT0);
323             // supportsFloatingPoint() && USE(JSVALUE64) => 3 SlowCases
324             addSlowCase(emitJumpIfNotImmediateNumber(regT0));
325             add64(tagTypeNumberRegister, regT0);
326             move64ToDouble(regT0, fpRegT0);
327             addSlowCase(branchTruncateDoubleToInt32(fpRegT0, regT0));
328             lhsIsInt.link(this);
329             emitJumpSlowCaseIfNotImmediateInteger(regT2);
330         } else {
331             // !supportsFloatingPoint() => 2 SlowCases
332             emitJumpSlowCaseIfNotImmediateInteger(regT0);
333             emitJumpSlowCaseIfNotImmediateInteger(regT2);
334         }
335         emitFastArithImmToInt(regT2);
336         urshift32(regT2, regT0);
337     }
338     emitFastArithIntToImmNoCheck(regT0, regT0);
339     emitPutVirtualRegister(result);
340 }
341
342 void JIT::emitSlow_op_urshift(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
343 {
344     int op2 = currentInstruction[3].u.operand;
345
346     if (isOperandConstantImmediateInt(op2))
347         linkSlowCase(iter);
348
349     else {
350         if (supportsFloatingPointTruncate()) {
351             linkSlowCase(iter);
352             linkSlowCase(iter);
353             linkSlowCase(iter);
354         } else {
355             linkSlowCase(iter);
356             linkSlowCase(iter);
357         }
358     }
359
360     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_urshift);
361     slowPathCall.call();
362 }
363
364 void JIT::emit_op_unsigned(Instruction* currentInstruction)
365 {
366     int result = currentInstruction[1].u.operand;
367     int op1 = currentInstruction[2].u.operand;
368     
369     emitGetVirtualRegister(op1, regT0);
370     emitJumpSlowCaseIfNotImmediateInteger(regT0);
371     addSlowCase(branch32(LessThan, regT0, TrustedImm32(0)));
372     emitFastArithReTagImmediate(regT0, regT0);
373     emitPutVirtualRegister(result, regT0);
374 }
375
376 void JIT::emitSlow_op_unsigned(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
377 {
378     linkSlowCase(iter);
379     linkSlowCase(iter);
380     
381     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_unsigned);
382     slowPathCall.call();
383 }
384
385 void JIT::emit_compareAndJump(OpcodeID, int op1, int op2, unsigned target, RelationalCondition condition)
386 {
387     // We generate inline code for the following cases in the fast path:
388     // - int immediate to constant int immediate
389     // - constant int immediate to int immediate
390     // - int immediate to int immediate
391
392     if (isOperandConstantImmediateChar(op1)) {
393         emitGetVirtualRegister(op2, regT0);
394         addSlowCase(emitJumpIfNotJSCell(regT0));
395         JumpList failures;
396         emitLoadCharacterString(regT0, regT0, failures);
397         addSlowCase(failures);
398         addJump(branch32(commute(condition), regT0, Imm32(asString(getConstantOperand(op1))->tryGetValue()[0])), target);
399         return;
400     }
401     if (isOperandConstantImmediateChar(op2)) {
402         emitGetVirtualRegister(op1, regT0);
403         addSlowCase(emitJumpIfNotJSCell(regT0));
404         JumpList failures;
405         emitLoadCharacterString(regT0, regT0, failures);
406         addSlowCase(failures);
407         addJump(branch32(condition, regT0, Imm32(asString(getConstantOperand(op2))->tryGetValue()[0])), target);
408         return;
409     }
410     if (isOperandConstantImmediateInt(op2)) {
411         emitGetVirtualRegister(op1, regT0);
412         emitJumpSlowCaseIfNotImmediateInteger(regT0);
413         int32_t op2imm = getConstantOperandImmediateInt(op2);
414         addJump(branch32(condition, regT0, Imm32(op2imm)), target);
415     } else if (isOperandConstantImmediateInt(op1)) {
416         emitGetVirtualRegister(op2, regT1);
417         emitJumpSlowCaseIfNotImmediateInteger(regT1);
418         int32_t op1imm = getConstantOperandImmediateInt(op1);
419         addJump(branch32(commute(condition), regT1, Imm32(op1imm)), target);
420     } else {
421         emitGetVirtualRegisters(op1, regT0, op2, regT1);
422         emitJumpSlowCaseIfNotImmediateInteger(regT0);
423         emitJumpSlowCaseIfNotImmediateInteger(regT1);
424
425         addJump(branch32(condition, regT0, regT1), target);
426     }
427 }
428
429 void JIT::emit_compareAndJumpSlow(int op1, int op2, unsigned target, DoubleCondition condition, size_t (JIT_OPERATION *operation)(ExecState*, EncodedJSValue, EncodedJSValue), bool invert, Vector<SlowCaseEntry>::iterator& iter)
430 {
431     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jlesseq), OPCODE_LENGTH_op_jlesseq_equals_op_jless);
432     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jnless), OPCODE_LENGTH_op_jnless_equals_op_jless);
433     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jnlesseq), OPCODE_LENGTH_op_jnlesseq_equals_op_jless);
434     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jgreater), OPCODE_LENGTH_op_jgreater_equals_op_jless);
435     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jgreatereq), OPCODE_LENGTH_op_jgreatereq_equals_op_jless);
436     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jngreater), OPCODE_LENGTH_op_jngreater_equals_op_jless);
437     COMPILE_ASSERT(OPCODE_LENGTH(op_jless) == OPCODE_LENGTH(op_jngreatereq), OPCODE_LENGTH_op_jngreatereq_equals_op_jless);
438     
439     // We generate inline code for the following cases in the slow path:
440     // - floating-point number to constant int immediate
441     // - constant int immediate to floating-point number
442     // - floating-point number to floating-point number.
443     if (isOperandConstantImmediateChar(op1) || isOperandConstantImmediateChar(op2)) {
444         linkSlowCase(iter);
445         linkSlowCase(iter);
446         linkSlowCase(iter);
447         linkSlowCase(iter);
448
449         emitGetVirtualRegister(op1, argumentGPR0);
450         emitGetVirtualRegister(op2, argumentGPR1);
451         callOperation(operation, argumentGPR0, argumentGPR1);
452         emitJumpSlowToHot(branchTest32(invert ? Zero : NonZero, returnValueGPR), target);
453         return;
454     }
455
456     if (isOperandConstantImmediateInt(op2)) {
457         linkSlowCase(iter);
458
459         if (supportsFloatingPoint()) {
460             Jump fail1 = emitJumpIfNotImmediateNumber(regT0);
461             add64(tagTypeNumberRegister, regT0);
462             move64ToDouble(regT0, fpRegT0);
463
464             int32_t op2imm = getConstantOperand(op2).asInt32();
465
466             move(Imm32(op2imm), regT1);
467             convertInt32ToDouble(regT1, fpRegT1);
468
469             emitJumpSlowToHot(branchDouble(condition, fpRegT0, fpRegT1), target);
470
471             emitJumpSlowToHot(jump(), OPCODE_LENGTH(op_jless));
472
473             fail1.link(this);
474         }
475
476         emitGetVirtualRegister(op2, regT1);
477         callOperation(operation, regT0, regT1);
478         emitJumpSlowToHot(branchTest32(invert ? Zero : NonZero, returnValueGPR), target);
479     } else if (isOperandConstantImmediateInt(op1)) {
480         linkSlowCase(iter);
481
482         if (supportsFloatingPoint()) {
483             Jump fail1 = emitJumpIfNotImmediateNumber(regT1);
484             add64(tagTypeNumberRegister, regT1);
485             move64ToDouble(regT1, fpRegT1);
486
487             int32_t op1imm = getConstantOperand(op1).asInt32();
488
489             move(Imm32(op1imm), regT0);
490             convertInt32ToDouble(regT0, fpRegT0);
491
492             emitJumpSlowToHot(branchDouble(condition, fpRegT0, fpRegT1), target);
493
494             emitJumpSlowToHot(jump(), OPCODE_LENGTH(op_jless));
495
496             fail1.link(this);
497         }
498
499         emitGetVirtualRegister(op1, regT2);
500         callOperation(operation, regT2, regT1);
501         emitJumpSlowToHot(branchTest32(invert ? Zero : NonZero, returnValueGPR), target);
502     } else {
503         linkSlowCase(iter);
504
505         if (supportsFloatingPoint()) {
506             Jump fail1 = emitJumpIfNotImmediateNumber(regT0);
507             Jump fail2 = emitJumpIfNotImmediateNumber(regT1);
508             Jump fail3 = emitJumpIfImmediateInteger(regT1);
509             add64(tagTypeNumberRegister, regT0);
510             add64(tagTypeNumberRegister, regT1);
511             move64ToDouble(regT0, fpRegT0);
512             move64ToDouble(regT1, fpRegT1);
513
514             emitJumpSlowToHot(branchDouble(condition, fpRegT0, fpRegT1), target);
515
516             emitJumpSlowToHot(jump(), OPCODE_LENGTH(op_jless));
517
518             fail1.link(this);
519             fail2.link(this);
520             fail3.link(this);
521         }
522
523         linkSlowCase(iter);
524         callOperation(operation, regT0, regT1);
525         emitJumpSlowToHot(branchTest32(invert ? Zero : NonZero, returnValueGPR), target);
526     }
527 }
528
529 void JIT::emit_op_bitand(Instruction* currentInstruction)
530 {
531     int result = currentInstruction[1].u.operand;
532     int op1 = currentInstruction[2].u.operand;
533     int op2 = currentInstruction[3].u.operand;
534
535     if (isOperandConstantImmediateInt(op1)) {
536         emitGetVirtualRegister(op2, regT0);
537         emitJumpSlowCaseIfNotImmediateInteger(regT0);
538         int32_t imm = getConstantOperandImmediateInt(op1);
539         and64(Imm32(imm), regT0);
540         if (imm >= 0)
541             emitFastArithIntToImmNoCheck(regT0, regT0);
542     } else if (isOperandConstantImmediateInt(op2)) {
543         emitGetVirtualRegister(op1, regT0);
544         emitJumpSlowCaseIfNotImmediateInteger(regT0);
545         int32_t imm = getConstantOperandImmediateInt(op2);
546         and64(Imm32(imm), regT0);
547         if (imm >= 0)
548             emitFastArithIntToImmNoCheck(regT0, regT0);
549     } else {
550         emitGetVirtualRegisters(op1, regT0, op2, regT1);
551         and64(regT1, regT0);
552         emitJumpSlowCaseIfNotImmediateInteger(regT0);
553     }
554     emitPutVirtualRegister(result);
555 }
556
557 void JIT::emitSlow_op_bitand(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
558 {
559     linkSlowCase(iter);
560
561     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_bitand);
562     slowPathCall.call();
563 }
564
565 void JIT::emit_op_inc(Instruction* currentInstruction)
566 {
567     int srcDst = currentInstruction[1].u.operand;
568
569     emitGetVirtualRegister(srcDst, regT0);
570     emitJumpSlowCaseIfNotImmediateInteger(regT0);
571     addSlowCase(branchAdd32(Overflow, TrustedImm32(1), regT0));
572     emitFastArithIntToImmNoCheck(regT0, regT0);
573     emitPutVirtualRegister(srcDst);
574 }
575
576 void JIT::emitSlow_op_inc(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
577 {
578     linkSlowCase(iter);
579     linkSlowCase(iter);
580     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_inc);
581     slowPathCall.call();
582 }
583
584 void JIT::emit_op_dec(Instruction* currentInstruction)
585 {
586     int srcDst = currentInstruction[1].u.operand;
587
588     emitGetVirtualRegister(srcDst, regT0);
589     emitJumpSlowCaseIfNotImmediateInteger(regT0);
590     addSlowCase(branchSub32(Overflow, TrustedImm32(1), regT0));
591     emitFastArithIntToImmNoCheck(regT0, regT0);
592     emitPutVirtualRegister(srcDst);
593 }
594
595 void JIT::emitSlow_op_dec(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
596 {
597     linkSlowCase(iter);
598     linkSlowCase(iter);
599     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_dec);
600     slowPathCall.call();
601 }
602
603 /* ------------------------------ BEGIN: OP_MOD ------------------------------ */
604
605 #if CPU(X86) || CPU(X86_64)
606
607 void JIT::emit_op_mod(Instruction* currentInstruction)
608 {
609     int result = currentInstruction[1].u.operand;
610     int op1 = currentInstruction[2].u.operand;
611     int op2 = currentInstruction[3].u.operand;
612
613     // Make sure registers are correct for x86 IDIV instructions.
614     ASSERT(regT0 == X86Registers::eax);
615     auto edx = X86Registers::edx;
616     auto ecx = X86Registers::ecx;
617     ASSERT(regT4 != edx);
618     ASSERT(regT4 != ecx);
619
620     emitGetVirtualRegisters(op1, regT4, op2, ecx);
621     emitJumpSlowCaseIfNotImmediateInteger(regT4);
622     emitJumpSlowCaseIfNotImmediateInteger(ecx);
623
624     move(regT4, regT0);
625     addSlowCase(branchTest32(Zero, ecx));
626     Jump denominatorNotNeg1 = branch32(NotEqual, ecx, TrustedImm32(-1));
627     addSlowCase(branch32(Equal, regT0, TrustedImm32(-2147483647-1)));
628     denominatorNotNeg1.link(this);
629     m_assembler.cdq();
630     m_assembler.idivl_r(ecx);
631     Jump numeratorPositive = branch32(GreaterThanOrEqual, regT4, TrustedImm32(0));
632     addSlowCase(branchTest32(Zero, edx));
633     numeratorPositive.link(this);
634     emitFastArithReTagImmediate(edx, regT0);
635     emitPutVirtualRegister(result);
636 }
637
638 void JIT::emitSlow_op_mod(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
639 {
640     linkSlowCase(iter);
641     linkSlowCase(iter);
642     linkSlowCase(iter);
643     linkSlowCase(iter);
644     linkSlowCase(iter);
645     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_mod);
646     slowPathCall.call();
647 }
648
649 #else // CPU(X86) || CPU(X86_64)
650
651 void JIT::emit_op_mod(Instruction* currentInstruction)
652 {
653     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_mod);
654     slowPathCall.call();
655 }
656
657 void JIT::emitSlow_op_mod(Instruction*, Vector<SlowCaseEntry>::iterator&)
658 {
659     UNREACHABLE_FOR_PLATFORM();
660 }
661
662 #endif // CPU(X86) || CPU(X86_64)
663
664 /* ------------------------------ END: OP_MOD ------------------------------ */
665
666 /* ------------------------------ BEGIN: USE(JSVALUE64) (OP_ADD, OP_SUB, OP_MUL) ------------------------------ */
667
668 void JIT::compileBinaryArithOp(OpcodeID opcodeID, int, int op1, int op2, OperandTypes)
669 {
670     emitGetVirtualRegisters(op1, regT0, op2, regT1);
671     emitJumpSlowCaseIfNotImmediateInteger(regT0);
672     emitJumpSlowCaseIfNotImmediateInteger(regT1);
673     RareCaseProfile* profile = m_codeBlock->addSpecialFastCaseProfile(m_bytecodeOffset);
674     if (opcodeID == op_add)
675         addSlowCase(branchAdd32(Overflow, regT1, regT0));
676     else if (opcodeID == op_sub)
677         addSlowCase(branchSub32(Overflow, regT1, regT0));
678     else {
679         ASSERT(opcodeID == op_mul);
680         if (shouldEmitProfiling()) {
681             // We want to be able to measure if this is taking the slow case just
682             // because of negative zero. If this produces positive zero, then we
683             // don't want the slow case to be taken because that will throw off
684             // speculative compilation.
685             move(regT0, regT2);
686             addSlowCase(branchMul32(Overflow, regT1, regT2));
687             JumpList done;
688             done.append(branchTest32(NonZero, regT2));
689             Jump negativeZero = branch32(LessThan, regT0, TrustedImm32(0));
690             done.append(branch32(GreaterThanOrEqual, regT1, TrustedImm32(0)));
691             negativeZero.link(this);
692             // We only get here if we have a genuine negative zero. Record this,
693             // so that the speculative JIT knows that we failed speculation
694             // because of a negative zero.
695             add32(TrustedImm32(1), AbsoluteAddress(&profile->m_counter));
696             addSlowCase(jump());
697             done.link(this);
698             move(regT2, regT0);
699         } else {
700             addSlowCase(branchMul32(Overflow, regT1, regT0));
701             addSlowCase(branchTest32(Zero, regT0));
702         }
703     }
704     emitFastArithIntToImmNoCheck(regT0, regT0);
705 }
706
707 void JIT::compileBinaryArithOpSlowCase(Instruction* currentInstruction, OpcodeID opcodeID, Vector<SlowCaseEntry>::iterator& iter, int result, int op1, int op2, OperandTypes types, bool op1HasImmediateIntFastCase, bool op2HasImmediateIntFastCase)
708 {
709     // We assume that subtracting TagTypeNumber is equivalent to adding DoubleEncodeOffset.
710     COMPILE_ASSERT(((TagTypeNumber + DoubleEncodeOffset) == 0), TagTypeNumber_PLUS_DoubleEncodeOffset_EQUALS_0);
711
712     Jump notImm1;
713     Jump notImm2;
714     if (op1HasImmediateIntFastCase) {
715         notImm2 = getSlowCase(iter);
716     } else if (op2HasImmediateIntFastCase) {
717         notImm1 = getSlowCase(iter);
718     } else {
719         notImm1 = getSlowCase(iter);
720         notImm2 = getSlowCase(iter);
721     }
722
723     linkSlowCase(iter); // Integer overflow case - we could handle this in JIT code, but this is likely rare.
724     if (opcodeID == op_mul && !op1HasImmediateIntFastCase && !op2HasImmediateIntFastCase) // op_mul has an extra slow case to handle 0 * negative number.
725         linkSlowCase(iter);
726
727     Label stubFunctionCall(this);
728
729     JITSlowPathCall slowPathCall(this, currentInstruction, opcodeID == op_add ? slow_path_add : opcodeID == op_sub ? slow_path_sub : slow_path_mul);
730     slowPathCall.call();
731     Jump end = jump();
732
733     if (op1HasImmediateIntFastCase) {
734         notImm2.link(this);
735         if (!types.second().definitelyIsNumber())
736             emitJumpIfNotImmediateNumber(regT0).linkTo(stubFunctionCall, this);
737         emitGetVirtualRegister(op1, regT1);
738         convertInt32ToDouble(regT1, fpRegT1);
739         add64(tagTypeNumberRegister, regT0);
740         move64ToDouble(regT0, fpRegT2);
741     } else if (op2HasImmediateIntFastCase) {
742         notImm1.link(this);
743         if (!types.first().definitelyIsNumber())
744             emitJumpIfNotImmediateNumber(regT0).linkTo(stubFunctionCall, this);
745         emitGetVirtualRegister(op2, regT1);
746         convertInt32ToDouble(regT1, fpRegT1);
747         add64(tagTypeNumberRegister, regT0);
748         move64ToDouble(regT0, fpRegT2);
749     } else {
750         // if we get here, eax is not an int32, edx not yet checked.
751         notImm1.link(this);
752         if (!types.first().definitelyIsNumber())
753             emitJumpIfNotImmediateNumber(regT0).linkTo(stubFunctionCall, this);
754         if (!types.second().definitelyIsNumber())
755             emitJumpIfNotImmediateNumber(regT1).linkTo(stubFunctionCall, this);
756         add64(tagTypeNumberRegister, regT0);
757         move64ToDouble(regT0, fpRegT1);
758         Jump op2isDouble = emitJumpIfNotImmediateInteger(regT1);
759         convertInt32ToDouble(regT1, fpRegT2);
760         Jump op2wasInteger = jump();
761
762         // if we get here, eax IS an int32, edx is not.
763         notImm2.link(this);
764         if (!types.second().definitelyIsNumber())
765             emitJumpIfNotImmediateNumber(regT1).linkTo(stubFunctionCall, this);
766         convertInt32ToDouble(regT0, fpRegT1);
767         op2isDouble.link(this);
768         add64(tagTypeNumberRegister, regT1);
769         move64ToDouble(regT1, fpRegT2);
770         op2wasInteger.link(this);
771     }
772
773     if (opcodeID == op_add)
774         addDouble(fpRegT2, fpRegT1);
775     else if (opcodeID == op_sub)
776         subDouble(fpRegT2, fpRegT1);
777     else if (opcodeID == op_mul)
778         mulDouble(fpRegT2, fpRegT1);
779     else {
780         ASSERT(opcodeID == op_div);
781         divDouble(fpRegT2, fpRegT1);
782     }
783     moveDoubleTo64(fpRegT1, regT0);
784     sub64(tagTypeNumberRegister, regT0);
785     emitPutVirtualRegister(result, regT0);
786
787     end.link(this);
788 }
789
790 void JIT::emit_op_add(Instruction* currentInstruction)
791 {
792     int result = currentInstruction[1].u.operand;
793     int op1 = currentInstruction[2].u.operand;
794     int op2 = currentInstruction[3].u.operand;
795     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
796
797     if (!types.first().mightBeNumber() || !types.second().mightBeNumber()) {
798         addSlowCase();
799         JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_add);
800         slowPathCall.call();
801         return;
802     }
803
804     if (isOperandConstantImmediateInt(op1)) {
805         emitGetVirtualRegister(op2, regT0);
806         emitJumpSlowCaseIfNotImmediateInteger(regT0);
807         addSlowCase(branchAdd32(Overflow, regT0, Imm32(getConstantOperandImmediateInt(op1)), regT1));
808         emitFastArithIntToImmNoCheck(regT1, regT0);
809     } else if (isOperandConstantImmediateInt(op2)) {
810         emitGetVirtualRegister(op1, regT0);
811         emitJumpSlowCaseIfNotImmediateInteger(regT0);
812         addSlowCase(branchAdd32(Overflow, regT0, Imm32(getConstantOperandImmediateInt(op2)), regT1));
813         emitFastArithIntToImmNoCheck(regT1, regT0);
814     } else
815         compileBinaryArithOp(op_add, result, op1, op2, types);
816
817     emitPutVirtualRegister(result);
818 }
819
820 void JIT::emitSlow_op_add(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
821 {
822     int result = currentInstruction[1].u.operand;
823     int op1 = currentInstruction[2].u.operand;
824     int op2 = currentInstruction[3].u.operand;
825     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
826
827     if (!types.first().mightBeNumber() || !types.second().mightBeNumber()) {
828         linkDummySlowCase(iter);
829         return;
830     }
831
832     bool op1HasImmediateIntFastCase = isOperandConstantImmediateInt(op1);
833     bool op2HasImmediateIntFastCase = !op1HasImmediateIntFastCase && isOperandConstantImmediateInt(op2);
834     compileBinaryArithOpSlowCase(currentInstruction, op_add, iter, result, op1, op2, types, op1HasImmediateIntFastCase, op2HasImmediateIntFastCase);
835 }
836
837 void JIT::emit_op_mul(Instruction* currentInstruction)
838 {
839     int result = currentInstruction[1].u.operand;
840     int op1 = currentInstruction[2].u.operand;
841     int op2 = currentInstruction[3].u.operand;
842     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
843
844     // For now, only plant a fast int case if the constant operand is greater than zero.
845     int32_t value;
846     if (isOperandConstantImmediateInt(op1) && ((value = getConstantOperandImmediateInt(op1)) > 0)) {
847         // Add a special fast case profile because the DFG JIT will expect one.
848         m_codeBlock->addSpecialFastCaseProfile(m_bytecodeOffset);
849         emitGetVirtualRegister(op2, regT0);
850         emitJumpSlowCaseIfNotImmediateInteger(regT0);
851         addSlowCase(branchMul32(Overflow, Imm32(value), regT0, regT1));
852         emitFastArithReTagImmediate(regT1, regT0);
853     } else if (isOperandConstantImmediateInt(op2) && ((value = getConstantOperandImmediateInt(op2)) > 0)) {
854         // Add a special fast case profile because the DFG JIT will expect one.
855         m_codeBlock->addSpecialFastCaseProfile(m_bytecodeOffset);
856         emitGetVirtualRegister(op1, regT0);
857         emitJumpSlowCaseIfNotImmediateInteger(regT0);
858         addSlowCase(branchMul32(Overflow, Imm32(value), regT0, regT1));
859         emitFastArithReTagImmediate(regT1, regT0);
860     } else
861         compileBinaryArithOp(op_mul, result, op1, op2, types);
862
863     emitPutVirtualRegister(result);
864 }
865
866 void JIT::emitSlow_op_mul(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
867 {
868     int result = currentInstruction[1].u.operand;
869     int op1 = currentInstruction[2].u.operand;
870     int op2 = currentInstruction[3].u.operand;
871     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
872
873     bool op1HasImmediateIntFastCase = isOperandConstantImmediateInt(op1) && getConstantOperandImmediateInt(op1) > 0;
874     bool op2HasImmediateIntFastCase = !op1HasImmediateIntFastCase && isOperandConstantImmediateInt(op2) && getConstantOperandImmediateInt(op2) > 0;
875     compileBinaryArithOpSlowCase(currentInstruction, op_mul, iter, result, op1, op2, types, op1HasImmediateIntFastCase, op2HasImmediateIntFastCase);
876 }
877
878 void JIT::emit_op_div(Instruction* currentInstruction)
879 {
880     int dst = currentInstruction[1].u.operand;
881     int op1 = currentInstruction[2].u.operand;
882     int op2 = currentInstruction[3].u.operand;
883     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
884
885     if (isOperandConstantImmediateDouble(op1)) {
886         emitGetVirtualRegister(op1, regT0);
887         add64(tagTypeNumberRegister, regT0);
888         move64ToDouble(regT0, fpRegT0);
889     } else if (isOperandConstantImmediateInt(op1)) {
890         emitLoadInt32ToDouble(op1, fpRegT0);
891     } else {
892         emitGetVirtualRegister(op1, regT0);
893         if (!types.first().definitelyIsNumber())
894             emitJumpSlowCaseIfNotImmediateNumber(regT0);
895         Jump notInt = emitJumpIfNotImmediateInteger(regT0);
896         convertInt32ToDouble(regT0, fpRegT0);
897         Jump skipDoubleLoad = jump();
898         notInt.link(this);
899         add64(tagTypeNumberRegister, regT0);
900         move64ToDouble(regT0, fpRegT0);
901         skipDoubleLoad.link(this);
902     }
903
904     if (isOperandConstantImmediateDouble(op2)) {
905         emitGetVirtualRegister(op2, regT1);
906         add64(tagTypeNumberRegister, regT1);
907         move64ToDouble(regT1, fpRegT1);
908     } else if (isOperandConstantImmediateInt(op2)) {
909         emitLoadInt32ToDouble(op2, fpRegT1);
910     } else {
911         emitGetVirtualRegister(op2, regT1);
912         if (!types.second().definitelyIsNumber())
913             emitJumpSlowCaseIfNotImmediateNumber(regT1);
914         Jump notInt = emitJumpIfNotImmediateInteger(regT1);
915         convertInt32ToDouble(regT1, fpRegT1);
916         Jump skipDoubleLoad = jump();
917         notInt.link(this);
918         add64(tagTypeNumberRegister, regT1);
919         move64ToDouble(regT1, fpRegT1);
920         skipDoubleLoad.link(this);
921     }
922     divDouble(fpRegT1, fpRegT0);
923     
924     // Is the result actually an integer? The DFG JIT would really like to know. If it's
925     // not an integer, we increment a count. If this together with the slow case counter
926     // are below threshold then the DFG JIT will compile this division with a specualtion
927     // that the remainder is zero.
928     
929     // As well, there are cases where a double result here would cause an important field
930     // in the heap to sometimes have doubles in it, resulting in double predictions getting
931     // propagated to a use site where it might cause damage (such as the index to an array
932     // access). So if we are DFG compiling anything in the program, we want this code to
933     // ensure that it produces integers whenever possible.
934     
935     JumpList notInteger;
936     branchConvertDoubleToInt32(fpRegT0, regT0, notInteger, fpRegT1);
937     // If we've got an integer, we might as well make that the result of the division.
938     emitFastArithReTagImmediate(regT0, regT0);
939     Jump isInteger = jump();
940     notInteger.link(this);
941     moveDoubleTo64(fpRegT0, regT0);
942     Jump doubleZero = branchTest64(Zero, regT0);
943     add32(TrustedImm32(1), AbsoluteAddress(&m_codeBlock->addSpecialFastCaseProfile(m_bytecodeOffset)->m_counter));
944     sub64(tagTypeNumberRegister, regT0);
945     Jump trueDouble = jump();
946     doubleZero.link(this);
947     move(tagTypeNumberRegister, regT0);
948     trueDouble.link(this);
949     isInteger.link(this);
950
951     emitPutVirtualRegister(dst, regT0);
952 }
953
954 void JIT::emitSlow_op_div(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
955 {
956     int op1 = currentInstruction[2].u.operand;
957     int op2 = currentInstruction[3].u.operand;
958     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
959     if (types.first().definitelyIsNumber() && types.second().definitelyIsNumber()) {
960         if (!ASSERT_DISABLED)
961             abortWithReason(JITDivOperandsAreNotNumbers);
962         return;
963     }
964     if (!isOperandConstantImmediateDouble(op1) && !isOperandConstantImmediateInt(op1)) {
965         if (!types.first().definitelyIsNumber())
966             linkSlowCase(iter);
967     }
968     if (!isOperandConstantImmediateDouble(op2) && !isOperandConstantImmediateInt(op2)) {
969         if (!types.second().definitelyIsNumber())
970             linkSlowCase(iter);
971     }
972     // There is an extra slow case for (op1 * -N) or (-N * op2), to check for 0 since this should produce a result of -0.
973     JITSlowPathCall slowPathCall(this, currentInstruction, slow_path_div);
974     slowPathCall.call();
975 }
976
977 void JIT::emit_op_sub(Instruction* currentInstruction)
978 {
979     int result = currentInstruction[1].u.operand;
980     int op1 = currentInstruction[2].u.operand;
981     int op2 = currentInstruction[3].u.operand;
982     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
983
984     compileBinaryArithOp(op_sub, result, op1, op2, types);
985     emitPutVirtualRegister(result);
986 }
987
988 void JIT::emitSlow_op_sub(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
989 {
990     int result = currentInstruction[1].u.operand;
991     int op1 = currentInstruction[2].u.operand;
992     int op2 = currentInstruction[3].u.operand;
993     OperandTypes types = OperandTypes::fromInt(currentInstruction[4].u.operand);
994
995     compileBinaryArithOpSlowCase(currentInstruction, op_sub, iter, result, op1, op2, types, false, false);
996 }
997
998 /* ------------------------------ END: OP_ADD, OP_SUB, OP_MUL ------------------------------ */
999
1000 #endif // USE(JSVALUE64)
1001
1002 } // namespace JSC
1003
1004 #endif // ENABLE(JIT)