Unreviewed, rolling out r192876.
[WebKit-https.git] / Source / JavaScriptCore / llint / LowLevelInterpreter64.asm
1 # Copyright (C) 2011-2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1. Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2. Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
13 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
14 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
16 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
17 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
18 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
19 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
20 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
22 # THE POSSIBILITY OF SUCH DAMAGE.
23
24
25 # Utilities.
26 macro jumpToInstruction()
27     jmp [PB, PC, 8]
28 end
29
30 macro dispatch(advance)
31     addp advance, PC
32     jumpToInstruction()
33 end
34
35 macro dispatchInt(advance)
36     addi advance, PC
37     jumpToInstruction()
38 end
39
40 macro dispatchIntIndirect(offset)
41     dispatchInt(offset * 8[PB, PC, 8])
42 end
43
44 macro dispatchAfterCall()
45     loadi ArgumentCount + TagOffset[cfr], PC
46     loadp CodeBlock[cfr], PB
47     loadp CodeBlock::m_instructions[PB], PB
48     loadisFromInstruction(1, t1)
49     storeq r0, [cfr, t1, 8]
50     valueProfile(r0, (CallOpCodeSize - 1), t3)
51     dispatch(CallOpCodeSize)
52 end
53
54 macro cCall2(function)
55     checkStackPointerAlignment(t4, 0xbad0c002)
56     if X86_64 or ARM64
57         call function
58     elsif X86_64_WIN
59         # Note: this implementation is only correct if the return type size is > 8 bytes.
60         # See macro cCall2Void for an implementation when the return type <= 8 bytes.
61         # On Win64, when the return type is larger than 8 bytes, we need to allocate space on the stack for the return value.
62         # On entry rcx (a0), should contain a pointer to this stack space. The other parameters are shifted to the right,
63         # rdx (a1) should contain the first argument, and r8 (a2) should contain the second argument.
64         # On return, rax contains a pointer to this stack value, and we then need to copy the 16 byte return value into rax (r0) and rdx (r1)
65         # since the return value is expected to be split between the two.
66         # See http://msdn.microsoft.com/en-us/library/7572ztz4.aspx
67         move a1, a2
68         move a0, a1
69         subp 48, sp
70         move sp, a0
71         addp 32, a0
72         call function
73         addp 48, sp
74         move 8[r0], r1
75         move [r0], r0
76     elsif C_LOOP
77         cloopCallSlowPath function, a0, a1
78     else
79         error
80     end
81 end
82
83 macro cCall2Void(function)
84     if C_LOOP
85         cloopCallSlowPathVoid function, a0, a1
86     elsif X86_64_WIN
87         # Note: we cannot use the cCall2 macro for Win64 in this case,
88         # as the Win64 cCall2 implemenation is only correct when the return type size is > 8 bytes.
89         # On Win64, rcx and rdx are used for passing the first two parameters.
90         # We also need to make room on the stack for all four parameter registers.
91         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
92         subp 32, sp 
93         call function
94         addp 32, sp 
95     else
96         cCall2(function)
97     end
98 end
99
100 # This barely works. arg3 and arg4 should probably be immediates.
101 macro cCall4(function)
102     checkStackPointerAlignment(t4, 0xbad0c004)
103     if X86_64 or ARM64
104         call function
105     elsif X86_64_WIN
106         # On Win64, rcx, rdx, r8, and r9 are used for passing the first four parameters.
107         # We also need to make room on the stack for all four parameter registers.
108         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
109         subp 64, sp
110         call function
111         addp 64, sp
112     else
113         error
114     end
115 end
116
117 macro doVMEntry(makeCall)
118     functionPrologue()
119     pushCalleeSaves()
120
121     const entry = a0
122     const vm = a1
123     const protoCallFrame = a2
124
125     vmEntryRecord(cfr, sp)
126
127     checkStackPointerAlignment(t4, 0xbad0dc01)
128
129     storep vm, VMEntryRecord::m_vm[sp]
130     loadp VM::topCallFrame[vm], t4
131     storep t4, VMEntryRecord::m_prevTopCallFrame[sp]
132     loadp VM::topVMEntryFrame[vm], t4
133     storep t4, VMEntryRecord::m_prevTopVMEntryFrame[sp]
134
135     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t4
136     addp CallFrameHeaderSlots, t4, t4
137     lshiftp 3, t4
138     subp sp, t4, t3
139
140     # Ensure that we have enough additional stack capacity for the incoming args,
141     # and the frame for the JS code we're executing. We need to do this check
142     # before we start copying the args from the protoCallFrame below.
143     bpaeq t3, VM::m_jsStackLimit[vm], .stackHeightOK
144
145     if C_LOOP
146         move entry, t4
147         move vm, t5
148         cloopCallSlowPath _llint_stack_check_at_vm_entry, vm, t3
149         bpeq t0, 0, .stackCheckFailed
150         move t4, entry
151         move t5, vm
152         jmp .stackHeightOK
153
154 .stackCheckFailed:
155         move t4, entry
156         move t5, vm
157     end
158
159     move vm, a0
160     move protoCallFrame, a1
161     cCall2(_llint_throw_stack_overflow_error)
162
163     vmEntryRecord(cfr, t4)
164
165     loadp VMEntryRecord::m_vm[t4], vm
166     loadp VMEntryRecord::m_prevTopCallFrame[t4], extraTempReg
167     storep extraTempReg, VM::topCallFrame[vm]
168     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], extraTempReg
169     storep extraTempReg, VM::topVMEntryFrame[vm]
170
171     subp cfr, CalleeRegisterSaveSize, sp
172
173     popCalleeSaves()
174     functionEpilogue()
175     ret
176
177 .stackHeightOK:
178     move t3, sp
179     move 4, t3
180
181 .copyHeaderLoop:
182     subi 1, t3
183     loadq [protoCallFrame, t3, 8], extraTempReg
184     storeq extraTempReg, CodeBlock[sp, t3, 8]
185     btinz t3, .copyHeaderLoop
186
187     loadi PayloadOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], t4
188     subi 1, t4
189     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], extraTempReg
190     subi 1, extraTempReg
191
192     bieq t4, extraTempReg, .copyArgs
193     move ValueUndefined, t3
194 .fillExtraArgsLoop:
195     subi 1, extraTempReg
196     storeq t3, ThisArgumentOffset + 8[sp, extraTempReg, 8]
197     bineq t4, extraTempReg, .fillExtraArgsLoop
198
199 .copyArgs:
200     loadp ProtoCallFrame::args[protoCallFrame], t3
201
202 .copyArgsLoop:
203     btiz t4, .copyArgsDone
204     subi 1, t4
205     loadq [t3, t4, 8], extraTempReg
206     storeq extraTempReg, ThisArgumentOffset + 8[sp, t4, 8]
207     jmp .copyArgsLoop
208
209 .copyArgsDone:
210     if ARM64
211         move sp, t4
212         storep t4, VM::topCallFrame[vm]
213     else
214         storep sp, VM::topCallFrame[vm]
215     end
216     storep cfr, VM::topVMEntryFrame[vm]
217
218     checkStackPointerAlignment(extraTempReg, 0xbad0dc02)
219
220     makeCall(entry, t3)
221
222     # We may have just made a call into a JS function, so we can't rely on sp
223     # for anything but the fact that our own locals (ie the VMEntryRecord) are
224     # not below it. It also still has to be aligned, though.
225     checkStackPointerAlignment(t2, 0xbad0dc03)
226
227     vmEntryRecord(cfr, t4)
228
229     loadp VMEntryRecord::m_vm[t4], vm
230     loadp VMEntryRecord::m_prevTopCallFrame[t4], t2
231     storep t2, VM::topCallFrame[vm]
232     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], t2
233     storep t2, VM::topVMEntryFrame[vm]
234
235     subp cfr, CalleeRegisterSaveSize, sp
236
237     popCalleeSaves()
238     functionEpilogue()
239
240     ret
241 end
242
243
244 macro makeJavaScriptCall(entry, temp)
245     addp 16, sp
246     if C_LOOP
247         cloopCallJSFunction entry
248     else
249         call entry
250     end
251     subp 16, sp
252 end
253
254
255 macro makeHostFunctionCall(entry, temp)
256     move entry, temp
257     storep cfr, [sp]
258     move sp, a0
259     if C_LOOP
260         storep lr, 8[sp]
261         cloopCallNative temp
262     elsif X86_64_WIN
263         # We need to allocate 32 bytes on the stack for the shadow space.
264         subp 32, sp
265         call temp
266         addp 32, sp
267     else
268         call temp
269     end
270 end
271
272
273 _handleUncaughtException:
274     loadp Callee[cfr], t3
275     andp MarkedBlockMask, t3
276     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
277     restoreCalleeSavesFromVMCalleeSavesBuffer(t3, t0)
278     loadp VM::callFrameForCatch[t3], cfr
279     storep 0, VM::callFrameForCatch[t3]
280
281     loadp CallerFrame[cfr], cfr
282     vmEntryRecord(cfr, t2)
283
284     loadp VMEntryRecord::m_vm[t2], t3
285     loadp VMEntryRecord::m_prevTopCallFrame[t2], extraTempReg
286     storep extraTempReg, VM::topCallFrame[t3]
287     loadp VMEntryRecord::m_prevTopVMEntryFrame[t2], extraTempReg
288     storep extraTempReg, VM::topVMEntryFrame[t3]
289
290     subp cfr, CalleeRegisterSaveSize, sp
291
292     popCalleeSaves()
293     functionEpilogue()
294     ret
295
296
297 macro prepareStateForCCall()
298     leap [PB, PC, 8], PC
299 end
300
301 macro restoreStateAfterCCall()
302     move r0, PC
303     subp PB, PC
304     rshiftp 3, PC
305 end
306
307 macro callSlowPath(slowPath)
308     prepareStateForCCall()
309     move cfr, a0
310     move PC, a1
311     cCall2(slowPath)
312     restoreStateAfterCCall()
313 end
314
315 macro traceOperand(fromWhere, operand)
316     prepareStateForCCall()
317     move fromWhere, a2
318     move operand, a3
319     move cfr, a0
320     move PC, a1
321     cCall4(_llint_trace_operand)
322     restoreStateAfterCCall()
323 end
324
325 macro traceValue(fromWhere, operand)
326     prepareStateForCCall()
327     move fromWhere, a2
328     move operand, a3
329     move cfr, a0
330     move PC, a1
331     cCall4(_llint_trace_value)
332     restoreStateAfterCCall()
333 end
334
335 # Call a slow path for call call opcodes.
336 macro callCallSlowPath(slowPath, action)
337     storei PC, ArgumentCount + TagOffset[cfr]
338     prepareStateForCCall()
339     move cfr, a0
340     move PC, a1
341     cCall2(slowPath)
342     action(r0, r1)
343 end
344
345 macro callWatchdogTimerHandler(throwHandler)
346     storei PC, ArgumentCount + TagOffset[cfr]
347     prepareStateForCCall()
348     move cfr, a0
349     move PC, a1
350     cCall2(_llint_slow_path_handle_watchdog_timer)
351     btpnz r0, throwHandler
352     loadi ArgumentCount + TagOffset[cfr], PC
353 end
354
355 macro checkSwitchToJITForLoop()
356     checkSwitchToJIT(
357         1,
358         macro()
359             storei PC, ArgumentCount + TagOffset[cfr]
360             prepareStateForCCall()
361             move cfr, a0
362             move PC, a1
363             cCall2(_llint_loop_osr)
364             btpz r0, .recover
365             move r1, sp
366             jmp r0
367         .recover:
368             loadi ArgumentCount + TagOffset[cfr], PC
369         end)
370 end
371
372 macro loadVariable(operand, value)
373     loadisFromInstruction(operand, value)
374     loadq [cfr, value, 8], value
375 end
376
377 # Index and value must be different registers. Index may be clobbered.
378 macro loadConstantOrVariable(index, value)
379     bpgteq index, FirstConstantRegisterIndex, .constant
380     loadq [cfr, index, 8], value
381     jmp .done
382 .constant:
383     loadp CodeBlock[cfr], value
384     loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value
385     subp FirstConstantRegisterIndex, index
386     loadq [value, index, 8], value
387 .done:
388 end
389
390 macro loadConstantOrVariableInt32(index, value, slow)
391     loadConstantOrVariable(index, value)
392     bqb value, tagTypeNumber, slow
393 end
394
395 macro loadConstantOrVariableCell(index, value, slow)
396     loadConstantOrVariable(index, value)
397     btqnz value, tagMask, slow
398 end
399
400 macro writeBarrierOnOperand(cellOperand)
401     loadisFromInstruction(cellOperand, t1)
402     loadConstantOrVariableCell(t1, t2, .writeBarrierDone)
403     skipIfIsRememberedOrInEden(t2, t1, t3, 
404         macro(cellState)
405             btbnz cellState, .writeBarrierDone
406             push PB, PC
407             move t2, a1 # t2 can be a0 (not on 64 bits, but better safe than sorry)
408             move cfr, a0
409             cCall2Void(_llint_write_barrier_slow)
410             pop PC, PB
411         end
412     )
413 .writeBarrierDone:
414 end
415
416 macro writeBarrierOnOperands(cellOperand, valueOperand)
417     loadisFromInstruction(valueOperand, t1)
418     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
419     btpz t0, .writeBarrierDone
420
421     writeBarrierOnOperand(cellOperand)
422 .writeBarrierDone:
423 end
424
425 macro writeBarrierOnGlobal(valueOperand, loadHelper)
426     loadisFromInstruction(valueOperand, t1)
427     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
428     btpz t0, .writeBarrierDone
429
430     loadHelper(t3)
431     skipIfIsRememberedOrInEden(t3, t1, t2,
432         macro(gcData)
433             btbnz gcData, .writeBarrierDone
434             push PB, PC
435             move cfr, a0
436             move t3, a1
437             cCall2Void(_llint_write_barrier_slow)
438             pop PC, PB
439         end
440     )
441 .writeBarrierDone:
442 end
443
444 macro writeBarrierOnGlobalObject(valueOperand)
445     writeBarrierOnGlobal(valueOperand,
446         macro(registerToStoreGlobal)
447             loadp CodeBlock[cfr], registerToStoreGlobal
448             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
449         end)
450 end
451
452 macro writeBarrierOnGlobalLexicalEnvironment(valueOperand)
453     writeBarrierOnGlobal(valueOperand,
454         macro(registerToStoreGlobal)
455             loadp CodeBlock[cfr], registerToStoreGlobal
456             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
457             loadp JSGlobalObject::m_globalLexicalEnvironment[registerToStoreGlobal], registerToStoreGlobal
458         end)
459 end
460
461 macro valueProfile(value, operand, scratch)
462     loadpFromInstruction(operand, scratch)
463     storeq value, ValueProfile::m_buckets[scratch]
464 end
465
466 macro structureIDToStructureWithScratch(structureIDThenStructure, scratch)
467     loadp CodeBlock[cfr], scratch
468     loadp CodeBlock::m_vm[scratch], scratch
469     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[scratch], scratch
470     loadp [scratch, structureIDThenStructure, 8], structureIDThenStructure
471 end
472
473 macro loadStructureWithScratch(cell, structure, scratch)
474     loadi JSCell::m_structureID[cell], structure
475     structureIDToStructureWithScratch(structure, scratch)
476 end
477
478 macro loadStructureAndClobberFirstArg(cell, structure)
479     loadi JSCell::m_structureID[cell], structure
480     loadp CodeBlock[cfr], cell
481     loadp CodeBlock::m_vm[cell], cell
482     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[cell], cell
483     loadp [cell, structure, 8], structure
484 end
485
486 macro storeStructureWithTypeInfo(cell, structure, scratch)
487     loadq Structure::m_blob + StructureIDBlob::u.doubleWord[structure], scratch
488     storeq scratch, JSCell::m_structureID[cell]
489 end
490
491 # Entrypoints into the interpreter.
492
493 # Expects that CodeBlock is in t1, which is what prologue() leaves behind.
494 macro functionArityCheck(doneLabel, slowPath)
495     loadi PayloadOffset + ArgumentCount[cfr], t0
496     biaeq t0, CodeBlock::m_numParameters[t1], doneLabel
497     prepareStateForCCall()
498     move cfr, a0
499     move PC, a1
500     cCall2(slowPath)   # This slowPath has the protocol: r0 = 0 => no error, r0 != 0 => error
501     btiz r0, .noError
502     move r1, cfr   # r1 contains caller frame
503     jmp _llint_throw_from_slow_path_trampoline
504
505 .noError:
506     loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[r1], t1
507     btiz t1, .continue
508     loadi PayloadOffset + ArgumentCount[cfr], t2
509     addi CallFrameHeaderSlots, t2
510
511     // Check if there are some unaligned slots we can use
512     move t1, t3
513     andi StackAlignmentSlots - 1, t3
514     btiz t3, .noExtraSlot
515     move ValueUndefined, t0
516 .fillExtraSlots:
517     storeq t0, [cfr, t2, 8]
518     addi 1, t2
519     bsubinz 1, t3, .fillExtraSlots
520     andi ~(StackAlignmentSlots - 1), t1
521     btiz t1, .continue
522
523 .noExtraSlot:
524     // Move frame up t1 slots
525     negq t1
526     move cfr, t3
527     subp CalleeSaveSpaceAsVirtualRegisters * 8, t3
528     addi CalleeSaveSpaceAsVirtualRegisters, t2
529 .copyLoop:
530     loadq [t3], t0
531     storeq t0, [t3, t1, 8]
532     addp 8, t3
533     bsubinz 1, t2, .copyLoop
534
535     // Fill new slots with JSUndefined
536     move t1, t2
537     move ValueUndefined, t0
538 .fillLoop:
539     storeq t0, [t3, t1, 8]
540     addp 8, t3
541     baddinz 1, t2, .fillLoop
542
543     lshiftp 3, t1
544     addp t1, cfr
545     addp t1, sp
546
547 .continue:
548     # Reload CodeBlock and reset PC, since the slow_path clobbered them.
549     loadp CodeBlock[cfr], t1
550     loadp CodeBlock::m_instructions[t1], PB
551     move 0, PC
552     jmp doneLabel
553 end
554
555 macro branchIfException(label)
556     loadp Callee[cfr], t3
557     andp MarkedBlockMask, t3
558     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
559     btqz VM::m_exception[t3], .noException
560     jmp label
561 .noException:
562 end
563
564
565 # Instruction implementations
566 _llint_op_enter:
567     traceExecution()
568     checkStackPointerAlignment(t2, 0xdead00e1)
569     loadp CodeBlock[cfr], t2                // t2<CodeBlock> = cfr.CodeBlock
570     loadi CodeBlock::m_numVars[t2], t2      // t2<size_t> = t2<CodeBlock>.m_numVars
571     subq CalleeSaveSpaceAsVirtualRegisters, t2
572     move cfr, t1
573     subq CalleeSaveSpaceAsVirtualRegisters * 8, t1
574     btiz t2, .opEnterDone
575     move ValueUndefined, t0
576     negi t2
577     sxi2q t2, t2
578 .opEnterLoop:
579     storeq t0, [t1, t2, 8]
580     addq 1, t2
581     btqnz t2, .opEnterLoop
582 .opEnterDone:
583     callSlowPath(_slow_path_enter)
584     dispatch(1)
585
586
587 _llint_op_get_scope:
588     traceExecution()
589     loadp Callee[cfr], t0
590     loadp JSCallee::m_scope[t0], t0
591     loadisFromInstruction(1, t1)
592     storeq t0, [cfr, t1, 8]
593     dispatch(2)
594
595
596 _llint_op_create_this:
597     traceExecution()
598     loadisFromInstruction(2, t0)
599     loadp [cfr, t0, 8], t0
600     loadp JSFunction::m_rareData[t0], t3
601     btpz t3, .opCreateThisSlow
602     loadp FunctionRareData::m_allocationProfile + ObjectAllocationProfile::m_allocator[t3], t1
603     loadp FunctionRareData::m_allocationProfile + ObjectAllocationProfile::m_structure[t3], t2
604     btpz t1, .opCreateThisSlow
605     loadpFromInstruction(4, t3)
606     bpeq t3, 1, .hasSeenMultipleCallee
607     bpneq t3, t0, .opCreateThisSlow
608 .hasSeenMultipleCallee:
609     allocateJSObject(t1, t2, t0, t3, .opCreateThisSlow)
610     loadisFromInstruction(1, t1)
611     storeq t0, [cfr, t1, 8]
612     dispatch(5)
613
614 .opCreateThisSlow:
615     callSlowPath(_slow_path_create_this)
616     dispatch(5)
617
618
619 _llint_op_to_this:
620     traceExecution()
621     loadisFromInstruction(1, t0)
622     loadq [cfr, t0, 8], t0
623     btqnz t0, tagMask, .opToThisSlow
624     bbneq JSCell::m_type[t0], FinalObjectType, .opToThisSlow
625     loadStructureWithScratch(t0, t1, t2)
626     loadpFromInstruction(2, t2)
627     bpneq t1, t2, .opToThisSlow
628     dispatch(4)
629
630 .opToThisSlow:
631     callSlowPath(_slow_path_to_this)
632     dispatch(4)
633
634
635 _llint_op_new_object:
636     traceExecution()
637     loadpFromInstruction(3, t0)
638     loadp ObjectAllocationProfile::m_allocator[t0], t1
639     loadp ObjectAllocationProfile::m_structure[t0], t2
640     allocateJSObject(t1, t2, t0, t3, .opNewObjectSlow)
641     loadisFromInstruction(1, t1)
642     storeq t0, [cfr, t1, 8]
643     dispatch(4)
644
645 .opNewObjectSlow:
646     callSlowPath(_llint_slow_path_new_object)
647     dispatch(4)
648
649
650 _llint_op_check_tdz:
651     traceExecution()
652     loadisFromInstruction(1, t0)
653     loadConstantOrVariable(t0, t1)
654     bqneq t1, ValueEmpty, .opNotTDZ
655     callSlowPath(_slow_path_throw_tdz_error)
656
657 .opNotTDZ:
658     dispatch(2)
659
660
661 _llint_op_mov:
662     traceExecution()
663     loadisFromInstruction(2, t1)
664     loadisFromInstruction(1, t0)
665     loadConstantOrVariable(t1, t2)
666     storeq t2, [cfr, t0, 8]
667     dispatch(3)
668
669
670 _llint_op_not:
671     traceExecution()
672     loadisFromInstruction(2, t0)
673     loadisFromInstruction(1, t1)
674     loadConstantOrVariable(t0, t2)
675     xorq ValueFalse, t2
676     btqnz t2, ~1, .opNotSlow
677     xorq ValueTrue, t2
678     storeq t2, [cfr, t1, 8]
679     dispatch(3)
680
681 .opNotSlow:
682     callSlowPath(_slow_path_not)
683     dispatch(3)
684
685
686 macro equalityComparison(integerComparison, slowPath)
687     traceExecution()
688     loadisFromInstruction(3, t0)
689     loadisFromInstruction(2, t2)
690     loadisFromInstruction(1, t3)
691     loadConstantOrVariableInt32(t0, t1, .slow)
692     loadConstantOrVariableInt32(t2, t0, .slow)
693     integerComparison(t0, t1, t0)
694     orq ValueFalse, t0
695     storeq t0, [cfr, t3, 8]
696     dispatch(4)
697
698 .slow:
699     callSlowPath(slowPath)
700     dispatch(4)
701 end
702
703 _llint_op_eq:
704     equalityComparison(
705         macro (left, right, result) cieq left, right, result end,
706         _slow_path_eq)
707
708
709 _llint_op_neq:
710     equalityComparison(
711         macro (left, right, result) cineq left, right, result end,
712         _slow_path_neq)
713
714
715 macro equalNullComparison()
716     loadisFromInstruction(2, t0)
717     loadq [cfr, t0, 8], t0
718     btqnz t0, tagMask, .immediate
719     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
720     move 0, t0
721     jmp .done
722 .masqueradesAsUndefined:
723     loadStructureWithScratch(t0, t2, t1)
724     loadp CodeBlock[cfr], t0
725     loadp CodeBlock::m_globalObject[t0], t0
726     cpeq Structure::m_globalObject[t2], t0, t0
727     jmp .done
728 .immediate:
729     andq ~TagBitUndefined, t0
730     cqeq t0, ValueNull, t0
731 .done:
732 end
733
734 _llint_op_eq_null:
735     traceExecution()
736     equalNullComparison()
737     loadisFromInstruction(1, t1)
738     orq ValueFalse, t0
739     storeq t0, [cfr, t1, 8]
740     dispatch(3)
741
742
743 _llint_op_neq_null:
744     traceExecution()
745     equalNullComparison()
746     loadisFromInstruction(1, t1)
747     xorq ValueTrue, t0
748     storeq t0, [cfr, t1, 8]
749     dispatch(3)
750
751
752 macro strictEq(equalityOperation, slowPath)
753     traceExecution()
754     loadisFromInstruction(3, t0)
755     loadisFromInstruction(2, t2)
756     loadConstantOrVariable(t0, t1)
757     loadConstantOrVariable(t2, t0)
758     move t0, t2
759     orq t1, t2
760     btqz t2, tagMask, .slow
761     bqaeq t0, tagTypeNumber, .leftOK
762     btqnz t0, tagTypeNumber, .slow
763 .leftOK:
764     bqaeq t1, tagTypeNumber, .rightOK
765     btqnz t1, tagTypeNumber, .slow
766 .rightOK:
767     equalityOperation(t0, t1, t0)
768     loadisFromInstruction(1, t1)
769     orq ValueFalse, t0
770     storeq t0, [cfr, t1, 8]
771     dispatch(4)
772
773 .slow:
774     callSlowPath(slowPath)
775     dispatch(4)
776 end
777
778 _llint_op_stricteq:
779     strictEq(
780         macro (left, right, result) cqeq left, right, result end,
781         _slow_path_stricteq)
782
783
784 _llint_op_nstricteq:
785     strictEq(
786         macro (left, right, result) cqneq left, right, result end,
787         _slow_path_nstricteq)
788
789
790 macro preOp(arithmeticOperation, slowPath)
791     traceExecution()
792     loadisFromInstruction(1, t0)
793     loadq [cfr, t0, 8], t1
794     bqb t1, tagTypeNumber, .slow
795     arithmeticOperation(t1, .slow)
796     orq tagTypeNumber, t1
797     storeq t1, [cfr, t0, 8]
798     dispatch(2)
799
800 .slow:
801     callSlowPath(slowPath)
802     dispatch(2)
803 end
804
805 _llint_op_inc:
806     preOp(
807         macro (value, slow) baddio 1, value, slow end,
808         _slow_path_inc)
809
810
811 _llint_op_dec:
812     preOp(
813         macro (value, slow) bsubio 1, value, slow end,
814         _slow_path_dec)
815
816
817 _llint_op_to_number:
818     traceExecution()
819     loadisFromInstruction(2, t0)
820     loadisFromInstruction(1, t1)
821     loadConstantOrVariable(t0, t2)
822     bqaeq t2, tagTypeNumber, .opToNumberIsImmediate
823     btqz t2, tagTypeNumber, .opToNumberSlow
824 .opToNumberIsImmediate:
825     storeq t2, [cfr, t1, 8]
826     dispatch(3)
827
828 .opToNumberSlow:
829     callSlowPath(_slow_path_to_number)
830     dispatch(3)
831
832
833 _llint_op_to_string:
834     traceExecution()
835     loadisFromInstruction(2, t1)
836     loadisFromInstruction(1, t2)
837     loadConstantOrVariable(t1, t0)
838     btqnz t0, tagMask, .opToStringSlow
839     bbneq JSCell::m_type[t0], StringType, .opToStringSlow
840 .opToStringIsString:
841     storeq t0, [cfr, t2, 8]
842     dispatch(3)
843
844 .opToStringSlow:
845     callSlowPath(_slow_path_to_string)
846     dispatch(3)
847
848
849 _llint_op_negate:
850     traceExecution()
851     loadisFromInstruction(2, t0)
852     loadisFromInstruction(1, t1)
853     loadConstantOrVariable(t0, t2)
854     bqb t2, tagTypeNumber, .opNegateNotInt
855     btiz t2, 0x7fffffff, .opNegateSlow
856     negi t2
857     orq tagTypeNumber, t2
858     storeq t2, [cfr, t1, 8]
859     dispatch(3)
860 .opNegateNotInt:
861     btqz t2, tagTypeNumber, .opNegateSlow
862     xorq 0x8000000000000000, t2
863     storeq t2, [cfr, t1, 8]
864     dispatch(3)
865
866 .opNegateSlow:
867     callSlowPath(_slow_path_negate)
868     dispatch(3)
869
870
871 macro binaryOpCustomStore(integerOperationAndStore, doubleOperation, slowPath)
872     loadisFromInstruction(3, t0)
873     loadisFromInstruction(2, t2)
874     loadConstantOrVariable(t0, t1)
875     loadConstantOrVariable(t2, t0)
876     bqb t0, tagTypeNumber, .op1NotInt
877     bqb t1, tagTypeNumber, .op2NotInt
878     loadisFromInstruction(1, t2)
879     integerOperationAndStore(t1, t0, .slow, t2)
880     dispatch(5)
881
882 .op1NotInt:
883     # First operand is definitely not an int, the second operand could be anything.
884     btqz t0, tagTypeNumber, .slow
885     bqaeq t1, tagTypeNumber, .op1NotIntOp2Int
886     btqz t1, tagTypeNumber, .slow
887     addq tagTypeNumber, t1
888     fq2d t1, ft1
889     jmp .op1NotIntReady
890 .op1NotIntOp2Int:
891     ci2d t1, ft1
892 .op1NotIntReady:
893     loadisFromInstruction(1, t2)
894     addq tagTypeNumber, t0
895     fq2d t0, ft0
896     doubleOperation(ft1, ft0)
897     fd2q ft0, t0
898     subq tagTypeNumber, t0
899     storeq t0, [cfr, t2, 8]
900     dispatch(5)
901
902 .op2NotInt:
903     # First operand is definitely an int, the second is definitely not.
904     loadisFromInstruction(1, t2)
905     btqz t1, tagTypeNumber, .slow
906     ci2d t0, ft0
907     addq tagTypeNumber, t1
908     fq2d t1, ft1
909     doubleOperation(ft1, ft0)
910     fd2q ft0, t0
911     subq tagTypeNumber, t0
912     storeq t0, [cfr, t2, 8]
913     dispatch(5)
914
915 .slow:
916     callSlowPath(slowPath)
917     dispatch(5)
918 end
919
920 macro binaryOp(integerOperation, doubleOperation, slowPath)
921     binaryOpCustomStore(
922         macro (left, right, slow, index)
923             integerOperation(left, right, slow)
924             orq tagTypeNumber, right
925             storeq right, [cfr, index, 8]
926         end,
927         doubleOperation, slowPath)
928 end
929
930 _llint_op_add:
931     traceExecution()
932     binaryOp(
933         macro (left, right, slow) baddio left, right, slow end,
934         macro (left, right) addd left, right end,
935         _slow_path_add)
936
937
938 _llint_op_mul:
939     traceExecution()
940     binaryOpCustomStore(
941         macro (left, right, slow, index)
942             # Assume t3 is scratchable.
943             move right, t3
944             bmulio left, t3, slow
945             btinz t3, .done
946             bilt left, 0, slow
947             bilt right, 0, slow
948         .done:
949             orq tagTypeNumber, t3
950             storeq t3, [cfr, index, 8]
951         end,
952         macro (left, right) muld left, right end,
953         _slow_path_mul)
954
955
956 _llint_op_sub:
957     traceExecution()
958     binaryOp(
959         macro (left, right, slow) bsubio left, right, slow end,
960         macro (left, right) subd left, right end,
961         _slow_path_sub)
962
963
964 _llint_op_div:
965     traceExecution()
966     if X86_64 or X86_64_WIN
967         binaryOpCustomStore(
968             macro (left, right, slow, index)
969                 # Assume t3 is scratchable.
970                 btiz left, slow
971                 bineq left, -1, .notNeg2TwoThe31DivByNeg1
972                 bieq right, -2147483648, .slow
973             .notNeg2TwoThe31DivByNeg1:
974                 btinz right, .intOK
975                 bilt left, 0, slow
976             .intOK:
977                 move left, t3
978                 move right, t0
979                 cdqi
980                 idivi t3
981                 btinz t1, slow
982                 orq tagTypeNumber, t0
983                 storeq t0, [cfr, index, 8]
984             end,
985             macro (left, right) divd left, right end,
986             _slow_path_div)
987     else
988         callSlowPath(_slow_path_div)
989         dispatch(5)
990     end
991
992
993 macro bitOp(operation, slowPath, advance)
994     loadisFromInstruction(3, t0)
995     loadisFromInstruction(2, t2)
996     loadisFromInstruction(1, t3)
997     loadConstantOrVariable(t0, t1)
998     loadConstantOrVariable(t2, t0)
999     bqb t0, tagTypeNumber, .slow
1000     bqb t1, tagTypeNumber, .slow
1001     operation(t1, t0)
1002     orq tagTypeNumber, t0
1003     storeq t0, [cfr, t3, 8]
1004     dispatch(advance)
1005
1006 .slow:
1007     callSlowPath(slowPath)
1008     dispatch(advance)
1009 end
1010
1011 _llint_op_lshift:
1012     traceExecution()
1013     bitOp(
1014         macro (left, right) lshifti left, right end,
1015         _slow_path_lshift,
1016         4)
1017
1018
1019 _llint_op_rshift:
1020     traceExecution()
1021     bitOp(
1022         macro (left, right) rshifti left, right end,
1023         _slow_path_rshift,
1024         4)
1025
1026
1027 _llint_op_urshift:
1028     traceExecution()
1029     bitOp(
1030         macro (left, right) urshifti left, right end,
1031         _slow_path_urshift,
1032         4)
1033
1034
1035 _llint_op_unsigned:
1036     traceExecution()
1037     loadisFromInstruction(1, t0)
1038     loadisFromInstruction(2, t1)
1039     loadConstantOrVariable(t1, t2)
1040     bilt t2, 0, .opUnsignedSlow
1041     storeq t2, [cfr, t0, 8]
1042     dispatch(3)
1043 .opUnsignedSlow:
1044     callSlowPath(_slow_path_unsigned)
1045     dispatch(3)
1046
1047
1048 _llint_op_bitand:
1049     traceExecution()
1050     bitOp(
1051         macro (left, right) andi left, right end,
1052         _slow_path_bitand,
1053         5)
1054
1055
1056 _llint_op_bitxor:
1057     traceExecution()
1058     bitOp(
1059         macro (left, right) xori left, right end,
1060         _slow_path_bitxor,
1061         5)
1062
1063
1064 _llint_op_bitor:
1065     traceExecution()
1066     bitOp(
1067         macro (left, right) ori left, right end,
1068         _slow_path_bitor,
1069         5)
1070
1071
1072 _llint_op_check_has_instance:
1073     traceExecution()
1074     loadisFromInstruction(3, t1)
1075     loadConstantOrVariableCell(t1, t0, .opCheckHasInstanceSlow)
1076     btbz JSCell::m_flags[t0], ImplementsDefaultHasInstance, .opCheckHasInstanceSlow
1077     dispatch(5)
1078
1079 .opCheckHasInstanceSlow:
1080     callSlowPath(_llint_slow_path_check_has_instance)
1081     dispatch(0)
1082
1083
1084 _llint_op_instanceof:
1085     traceExecution()
1086     # Actually do the work.
1087     loadisFromInstruction(3, t0)
1088     loadConstantOrVariableCell(t0, t1, .opInstanceofSlow)
1089     bbb JSCell::m_type[t1], ObjectType, .opInstanceofSlow
1090     loadisFromInstruction(2, t0)
1091     loadConstantOrVariableCell(t0, t2, .opInstanceofSlow)
1092     
1093     # Register state: t1 = prototype, t2 = value
1094     move 1, t0
1095 .opInstanceofLoop:
1096     loadStructureAndClobberFirstArg(t2, t3)
1097     loadq Structure::m_prototype[t3], t2
1098     bqeq t2, t1, .opInstanceofDone
1099     btqz t2, tagMask, .opInstanceofLoop
1100
1101     move 0, t0
1102 .opInstanceofDone:
1103     orq ValueFalse, t0
1104     loadisFromInstruction(1, t3)
1105     storeq t0, [cfr, t3, 8]
1106     dispatch(4)
1107
1108 .opInstanceofSlow:
1109     callSlowPath(_llint_slow_path_instanceof)
1110     dispatch(4)
1111
1112
1113 _llint_op_is_undefined:
1114     traceExecution()
1115     loadisFromInstruction(2, t1)
1116     loadisFromInstruction(1, t2)
1117     loadConstantOrVariable(t1, t0)
1118     btqz t0, tagMask, .opIsUndefinedCell
1119     cqeq t0, ValueUndefined, t3
1120     orq ValueFalse, t3
1121     storeq t3, [cfr, t2, 8]
1122     dispatch(3)
1123 .opIsUndefinedCell:
1124     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
1125     move ValueFalse, t1
1126     storeq t1, [cfr, t2, 8]
1127     dispatch(3)
1128 .masqueradesAsUndefined:
1129     loadStructureWithScratch(t0, t3, t1)
1130     loadp CodeBlock[cfr], t1
1131     loadp CodeBlock::m_globalObject[t1], t1
1132     cpeq Structure::m_globalObject[t3], t1, t0
1133     orq ValueFalse, t0
1134     storeq t0, [cfr, t2, 8]
1135     dispatch(3)
1136
1137
1138 _llint_op_is_boolean:
1139     traceExecution()
1140     loadisFromInstruction(2, t1)
1141     loadisFromInstruction(1, t2)
1142     loadConstantOrVariable(t1, t0)
1143     xorq ValueFalse, t0
1144     tqz t0, ~1, t0
1145     orq ValueFalse, t0
1146     storeq t0, [cfr, t2, 8]
1147     dispatch(3)
1148
1149
1150 _llint_op_is_number:
1151     traceExecution()
1152     loadisFromInstruction(2, t1)
1153     loadisFromInstruction(1, t2)
1154     loadConstantOrVariable(t1, t0)
1155     tqnz t0, tagTypeNumber, t1
1156     orq ValueFalse, t1
1157     storeq t1, [cfr, t2, 8]
1158     dispatch(3)
1159
1160
1161 _llint_op_is_string:
1162     traceExecution()
1163     loadisFromInstruction(2, t1)
1164     loadisFromInstruction(1, t2)
1165     loadConstantOrVariable(t1, t0)
1166     btqnz t0, tagMask, .opIsStringNotCell
1167     cbeq JSCell::m_type[t0], StringType, t1
1168     orq ValueFalse, t1
1169     storeq t1, [cfr, t2, 8]
1170     dispatch(3)
1171 .opIsStringNotCell:
1172     storeq ValueFalse, [cfr, t2, 8]
1173     dispatch(3)
1174
1175
1176 _llint_op_is_object:
1177     traceExecution()
1178     loadisFromInstruction(2, t1)
1179     loadisFromInstruction(1, t2)
1180     loadConstantOrVariable(t1, t0)
1181     btqnz t0, tagMask, .opIsObjectNotCell
1182     cbaeq JSCell::m_type[t0], ObjectType, t1
1183     orq ValueFalse, t1
1184     storeq t1, [cfr, t2, 8]
1185     dispatch(3)
1186 .opIsObjectNotCell:
1187     storeq ValueFalse, [cfr, t2, 8]
1188     dispatch(3)
1189
1190
1191 macro loadPropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value, slow)
1192     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1193     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1194     copyBarrier(objectAndStorage, slow)
1195     negi propertyOffsetAsInt
1196     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1197     jmp .ready
1198 .isInline:
1199     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1200 .ready:
1201     loadq (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8], value
1202 end
1203
1204
1205 macro storePropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value, slow)
1206     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1207     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1208     copyBarrier(objectAndStorage, slow)
1209     negi propertyOffsetAsInt
1210     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1211     jmp .ready
1212 .isInline:
1213     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1214 .ready:
1215     storeq value, (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8]
1216 end
1217
1218 _llint_op_get_by_id:
1219     traceExecution()
1220     loadisFromInstruction(2, t0)
1221     loadConstantOrVariableCell(t0, t3, .opGetByIdSlow)
1222     loadi JSCell::m_structureID[t3], t1
1223     loadisFromInstruction(4, t2)
1224     bineq t2, t1, .opGetByIdSlow
1225     loadisFromInstruction(5, t1)
1226     loadisFromInstruction(1, t2)
1227     loadPropertyAtVariableOffset(t1, t3, t0, .opGetByIdSlow)
1228     storeq t0, [cfr, t2, 8]
1229     valueProfile(t0, 8, t1)
1230     dispatch(9)
1231
1232 .opGetByIdSlow:
1233     callSlowPath(_llint_slow_path_get_by_id)
1234     dispatch(9)
1235
1236
1237 _llint_op_get_array_length:
1238     traceExecution()
1239     loadisFromInstruction(2, t0)
1240     loadpFromInstruction(4, t1)
1241     loadConstantOrVariableCell(t0, t3, .opGetArrayLengthSlow)
1242     move t3, t2
1243     arrayProfile(t2, t1, t0)
1244     btiz t2, IsArray, .opGetArrayLengthSlow
1245     btiz t2, IndexingShapeMask, .opGetArrayLengthSlow
1246     loadisFromInstruction(1, t1)
1247     loadp JSObject::m_butterfly[t3], t0
1248     copyBarrier(t0, .opGetArrayLengthSlow)
1249     loadi -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], t0
1250     bilt t0, 0, .opGetArrayLengthSlow
1251     orq tagTypeNumber, t0
1252     valueProfile(t0, 8, t2)
1253     storeq t0, [cfr, t1, 8]
1254     dispatch(9)
1255
1256 .opGetArrayLengthSlow:
1257     callSlowPath(_llint_slow_path_get_by_id)
1258     dispatch(9)
1259
1260
1261 _llint_op_put_by_id:
1262     traceExecution()
1263     writeBarrierOnOperands(1, 3)
1264     loadisFromInstruction(1, t3)
1265     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
1266     loadisFromInstruction(4, t2)
1267     bineq t2, JSCell::m_structureID[t0], .opPutByIdSlow
1268
1269     # At this point, we have:
1270     # t2 -> current structure ID
1271     # t0 -> object base
1272
1273     loadisFromInstruction(3, t1)
1274     loadConstantOrVariable(t1, t3)
1275
1276     loadpFromInstruction(8, t1)
1277
1278     # At this point, we have:
1279     # t0 -> object base
1280     # t1 -> put by id flags
1281     # t2 -> current structure ID
1282     # t3 -> value to put
1283
1284     btpnz t1, PutByIdPrimaryTypeMask, .opPutByIdTypeCheckObjectWithStructureOrOther
1285
1286     # We have one of the non-structure type checks. Find out which one.
1287     andp PutByIdSecondaryTypeMask, t1
1288     bplt t1, PutByIdSecondaryTypeString, .opPutByIdTypeCheckLessThanString
1289
1290     # We are one of the following: String, Symbol, Object, ObjectOrOther, Top
1291     bplt t1, PutByIdSecondaryTypeObjectOrOther, .opPutByIdTypeCheckLessThanObjectOrOther
1292
1293     # We are either ObjectOrOther or Top.
1294     bpeq t1, PutByIdSecondaryTypeTop, .opPutByIdDoneCheckingTypes
1295
1296     # Check if we are ObjectOrOther.
1297     btqz t3, tagMask, .opPutByIdTypeCheckObject
1298 .opPutByIdTypeCheckOther:
1299     andq ~TagBitUndefined, t3
1300     bqeq t3, ValueNull, .opPutByIdDoneCheckingTypes
1301     jmp .opPutByIdSlow
1302
1303 .opPutByIdTypeCheckLessThanObjectOrOther:
1304     # We are either String, Symbol or Object.
1305     btqnz t3, tagMask, .opPutByIdSlow
1306     bpeq t1, PutByIdSecondaryTypeObject, .opPutByIdTypeCheckObject
1307     bpeq t1, PutByIdSecondaryTypeSymbol, .opPutByIdTypeCheckSymbol
1308     bbeq JSCell::m_type[t3], StringType, .opPutByIdDoneCheckingTypes
1309     jmp .opPutByIdSlow
1310 .opPutByIdTypeCheckObject:
1311     bbaeq JSCell::m_type[t3], ObjectType, .opPutByIdDoneCheckingTypes
1312     jmp .opPutByIdSlow
1313 .opPutByIdTypeCheckSymbol:
1314     bbeq JSCell::m_type[t3], SymbolType, .opPutByIdDoneCheckingTypes
1315     jmp .opPutByIdSlow
1316
1317 .opPutByIdTypeCheckLessThanString:
1318     # We are one of the following: Bottom, Boolean, Other, Int32, Number
1319     bplt t1, PutByIdSecondaryTypeInt32, .opPutByIdTypeCheckLessThanInt32
1320
1321     # We are either Int32 or Number.
1322     bpeq t1, PutByIdSecondaryTypeNumber, .opPutByIdTypeCheckNumber
1323
1324     bqaeq t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1325     jmp .opPutByIdSlow
1326
1327 .opPutByIdTypeCheckNumber:
1328     btqnz t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1329     jmp .opPutByIdSlow
1330
1331 .opPutByIdTypeCheckLessThanInt32:
1332     # We are one of the following: Bottom, Boolean, Other.
1333     bpneq t1, PutByIdSecondaryTypeBoolean, .opPutByIdTypeCheckBottomOrOther
1334     xorq ValueFalse, t3
1335     btqz t3, ~1, .opPutByIdDoneCheckingTypes
1336     jmp .opPutByIdSlow
1337
1338 .opPutByIdTypeCheckBottomOrOther:
1339     bpeq t1, PutByIdSecondaryTypeOther, .opPutByIdTypeCheckOther
1340     jmp .opPutByIdSlow
1341
1342 .opPutByIdTypeCheckObjectWithStructureOrOther:
1343     btqz t3, tagMask, .opPutByIdTypeCheckObjectWithStructure
1344     btpnz t1, PutByIdPrimaryTypeObjectWithStructureOrOther, .opPutByIdTypeCheckOther
1345     jmp .opPutByIdSlow
1346
1347 .opPutByIdTypeCheckObjectWithStructure:
1348     urshiftp 3, t1
1349     bineq t1, JSCell::m_structureID[t3], .opPutByIdSlow
1350
1351 .opPutByIdDoneCheckingTypes:
1352     loadisFromInstruction(6, t1)
1353     
1354     btiz t1, .opPutByIdNotTransition
1355
1356     # This is the transition case. t1 holds the new structureID. t2 holds the old structure ID.
1357     # If we have a chain, we need to check it. t0 is the base. We may clobber t1 to use it as
1358     # scratch.
1359     loadpFromInstruction(7, t3)
1360     btpz t3, .opPutByIdTransitionDirect
1361
1362     loadp StructureChain::m_vector[t3], t3
1363     assert(macro (ok) btpnz t3, ok end)
1364
1365     structureIDToStructureWithScratch(t2, t1)
1366     loadq Structure::m_prototype[t2], t2
1367     bqeq t2, ValueNull, .opPutByIdTransitionChainDone
1368 .opPutByIdTransitionChainLoop:
1369     # At this point, t2 contains a prototye, and [t3] contains the Structure* that we want that
1370     # prototype to have. We don't want to have to load the Structure* for t2. Instead, we load
1371     # the Structure* from [t3], and then we compare its id to the id in the header of t2.
1372     loadp [t3], t1
1373     loadi JSCell::m_structureID[t2], t2
1374     # Now, t1 has the Structure* and t2 has the StructureID that we want that Structure* to have.
1375     bineq t2, Structure::m_blob + StructureIDBlob::u.fields.structureID[t1], .opPutByIdSlow
1376     addp 8, t3
1377     loadq Structure::m_prototype[t1], t2
1378     bqneq t2, ValueNull, .opPutByIdTransitionChainLoop
1379
1380 .opPutByIdTransitionChainDone:
1381     # Reload the new structure, since we clobbered it above.
1382     loadisFromInstruction(6, t1)
1383
1384 .opPutByIdTransitionDirect:
1385     storei t1, JSCell::m_structureID[t0]
1386
1387 .opPutByIdNotTransition:
1388     # The only thing live right now is t0, which holds the base.
1389     loadisFromInstruction(3, t1)
1390     loadConstantOrVariable(t1, t2)
1391     loadisFromInstruction(5, t1)
1392     storePropertyAtVariableOffset(t1, t0, t2, .opPutByIdSlow)
1393     dispatch(9)
1394
1395 .opPutByIdSlow:
1396     callSlowPath(_llint_slow_path_put_by_id)
1397     dispatch(9)
1398
1399
1400 _llint_op_get_by_val:
1401     traceExecution()
1402     loadisFromInstruction(2, t2)
1403     loadConstantOrVariableCell(t2, t0, .opGetByValSlow)
1404     loadpFromInstruction(4, t3)
1405     move t0, t2
1406     arrayProfile(t2, t3, t1)
1407     loadisFromInstruction(3, t3)
1408     loadConstantOrVariableInt32(t3, t1, .opGetByValSlow)
1409     sxi2q t1, t1
1410     loadp JSObject::m_butterfly[t0], t3
1411     copyBarrier(t3, .opGetByValSlow)
1412     andi IndexingShapeMask, t2
1413     bieq t2, Int32Shape, .opGetByValIsContiguous
1414     bineq t2, ContiguousShape, .opGetByValNotContiguous
1415 .opGetByValIsContiguous:
1416
1417     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1418     loadisFromInstruction(1, t0)
1419     loadq [t3, t1, 8], t2
1420     btqz t2, .opGetByValOutOfBounds
1421     jmp .opGetByValDone
1422
1423 .opGetByValNotContiguous:
1424     bineq t2, DoubleShape, .opGetByValNotDouble
1425     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1426     loadis 8[PB, PC, 8], t0
1427     loadd [t3, t1, 8], ft0
1428     bdnequn ft0, ft0, .opGetByValOutOfBounds
1429     fd2q ft0, t2
1430     subq tagTypeNumber, t2
1431     jmp .opGetByValDone
1432     
1433 .opGetByValNotDouble:
1434     subi ArrayStorageShape, t2
1435     bia t2, SlowPutArrayStorageShape - ArrayStorageShape, .opGetByValSlow
1436     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t3], .opGetByValOutOfBounds
1437     loadisFromInstruction(1, t0)
1438     loadq ArrayStorage::m_vector[t3, t1, 8], t2
1439     btqz t2, .opGetByValOutOfBounds
1440
1441 .opGetByValDone:
1442     storeq t2, [cfr, t0, 8]
1443     valueProfile(t2, 5, t0)
1444     dispatch(6)
1445
1446 .opGetByValOutOfBounds:
1447     loadpFromInstruction(4, t0)
1448     storeb 1, ArrayProfile::m_outOfBounds[t0]
1449 .opGetByValSlow:
1450     callSlowPath(_llint_slow_path_get_by_val)
1451     dispatch(6)
1452
1453
1454 macro contiguousPutByVal(storeCallback)
1455     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .outOfBounds
1456 .storeResult:
1457     loadisFromInstruction(3, t2)
1458     storeCallback(t2, t1, [t0, t3, 8])
1459     dispatch(5)
1460
1461 .outOfBounds:
1462     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1463     loadp 32[PB, PC, 8], t2
1464     storeb 1, ArrayProfile::m_mayStoreToHole[t2]
1465     addi 1, t3, t2
1466     storei t2, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1467     jmp .storeResult
1468 end
1469
1470 macro putByVal(slowPath)
1471     traceExecution()
1472     writeBarrierOnOperands(1, 3)
1473     loadisFromInstruction(1, t0)
1474     loadConstantOrVariableCell(t0, t1, .opPutByValSlow)
1475     loadpFromInstruction(4, t3)
1476     move t1, t2
1477     arrayProfile(t2, t3, t0)
1478     loadisFromInstruction(2, t0)
1479     loadConstantOrVariableInt32(t0, t3, .opPutByValSlow)
1480     sxi2q t3, t3
1481     loadp JSObject::m_butterfly[t1], t0
1482     copyBarrier(t0, .opPutByValSlow)
1483     andi IndexingShapeMask, t2
1484     bineq t2, Int32Shape, .opPutByValNotInt32
1485     contiguousPutByVal(
1486         macro (operand, scratch, address)
1487             loadConstantOrVariable(operand, scratch)
1488             bpb scratch, tagTypeNumber, .opPutByValSlow
1489             storep scratch, address
1490         end)
1491
1492 .opPutByValNotInt32:
1493     bineq t2, DoubleShape, .opPutByValNotDouble
1494     contiguousPutByVal(
1495         macro (operand, scratch, address)
1496             loadConstantOrVariable(operand, scratch)
1497             bqb scratch, tagTypeNumber, .notInt
1498             ci2d scratch, ft0
1499             jmp .ready
1500         .notInt:
1501             addp tagTypeNumber, scratch
1502             fq2d scratch, ft0
1503             bdnequn ft0, ft0, .opPutByValSlow
1504         .ready:
1505             stored ft0, address
1506         end)
1507
1508 .opPutByValNotDouble:
1509     bineq t2, ContiguousShape, .opPutByValNotContiguous
1510     contiguousPutByVal(
1511         macro (operand, scratch, address)
1512             loadConstantOrVariable(operand, scratch)
1513             storep scratch, address
1514         end)
1515
1516 .opPutByValNotContiguous:
1517     bineq t2, ArrayStorageShape, .opPutByValSlow
1518     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1519     btqz ArrayStorage::m_vector[t0, t3, 8], .opPutByValArrayStorageEmpty
1520 .opPutByValArrayStorageStoreResult:
1521     loadisFromInstruction(3, t2)
1522     loadConstantOrVariable(t2, t1)
1523     storeq t1, ArrayStorage::m_vector[t0, t3, 8]
1524     dispatch(5)
1525
1526 .opPutByValArrayStorageEmpty:
1527     loadpFromInstruction(4, t1)
1528     storeb 1, ArrayProfile::m_mayStoreToHole[t1]
1529     addi 1, ArrayStorage::m_numValuesInVector[t0]
1530     bib t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .opPutByValArrayStorageStoreResult
1531     addi 1, t3, t1
1532     storei t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1533     jmp .opPutByValArrayStorageStoreResult
1534
1535 .opPutByValOutOfBounds:
1536     loadpFromInstruction(4, t0)
1537     storeb 1, ArrayProfile::m_outOfBounds[t0]
1538 .opPutByValSlow:
1539     callSlowPath(slowPath)
1540     dispatch(5)
1541 end
1542
1543 _llint_op_put_by_val:
1544     putByVal(_llint_slow_path_put_by_val)
1545
1546 _llint_op_put_by_val_direct:
1547     putByVal(_llint_slow_path_put_by_val_direct)
1548
1549
1550 _llint_op_jmp:
1551     traceExecution()
1552     dispatchIntIndirect(1)
1553
1554
1555 macro jumpTrueOrFalse(conditionOp, slow)
1556     loadisFromInstruction(1, t1)
1557     loadConstantOrVariable(t1, t0)
1558     xorq ValueFalse, t0
1559     btqnz t0, -1, .slow
1560     conditionOp(t0, .target)
1561     dispatch(3)
1562
1563 .target:
1564     dispatchIntIndirect(2)
1565
1566 .slow:
1567     callSlowPath(slow)
1568     dispatch(0)
1569 end
1570
1571
1572 macro equalNull(cellHandler, immediateHandler)
1573     loadisFromInstruction(1, t0)
1574     assertNotConstant(t0)
1575     loadq [cfr, t0, 8], t0
1576     btqnz t0, tagMask, .immediate
1577     loadStructureWithScratch(t0, t2, t1)
1578     cellHandler(t2, JSCell::m_flags[t0], .target)
1579     dispatch(3)
1580
1581 .target:
1582     dispatchIntIndirect(2)
1583
1584 .immediate:
1585     andq ~TagBitUndefined, t0
1586     immediateHandler(t0, .target)
1587     dispatch(3)
1588 end
1589
1590 _llint_op_jeq_null:
1591     traceExecution()
1592     equalNull(
1593         macro (structure, value, target) 
1594             btbz value, MasqueradesAsUndefined, .notMasqueradesAsUndefined
1595             loadp CodeBlock[cfr], t0
1596             loadp CodeBlock::m_globalObject[t0], t0
1597             bpeq Structure::m_globalObject[structure], t0, target
1598 .notMasqueradesAsUndefined:
1599         end,
1600         macro (value, target) bqeq value, ValueNull, target end)
1601
1602
1603 _llint_op_jneq_null:
1604     traceExecution()
1605     equalNull(
1606         macro (structure, value, target) 
1607             btbz value, MasqueradesAsUndefined, target
1608             loadp CodeBlock[cfr], t0
1609             loadp CodeBlock::m_globalObject[t0], t0
1610             bpneq Structure::m_globalObject[structure], t0, target
1611         end,
1612         macro (value, target) bqneq value, ValueNull, target end)
1613
1614
1615 _llint_op_jneq_ptr:
1616     traceExecution()
1617     loadisFromInstruction(1, t0)
1618     loadisFromInstruction(2, t1)
1619     loadp CodeBlock[cfr], t2
1620     loadp CodeBlock::m_globalObject[t2], t2
1621     loadp JSGlobalObject::m_specialPointers[t2, t1, 8], t1
1622     bpneq t1, [cfr, t0, 8], .opJneqPtrTarget
1623     dispatch(4)
1624
1625 .opJneqPtrTarget:
1626     dispatchIntIndirect(3)
1627
1628
1629 macro compare(integerCompare, doubleCompare, slowPath)
1630     loadisFromInstruction(1, t2)
1631     loadisFromInstruction(2, t3)
1632     loadConstantOrVariable(t2, t0)
1633     loadConstantOrVariable(t3, t1)
1634     bqb t0, tagTypeNumber, .op1NotInt
1635     bqb t1, tagTypeNumber, .op2NotInt
1636     integerCompare(t0, t1, .jumpTarget)
1637     dispatch(4)
1638
1639 .op1NotInt:
1640     btqz t0, tagTypeNumber, .slow
1641     bqb t1, tagTypeNumber, .op1NotIntOp2NotInt
1642     ci2d t1, ft1
1643     jmp .op1NotIntReady
1644 .op1NotIntOp2NotInt:
1645     btqz t1, tagTypeNumber, .slow
1646     addq tagTypeNumber, t1
1647     fq2d t1, ft1
1648 .op1NotIntReady:
1649     addq tagTypeNumber, t0
1650     fq2d t0, ft0
1651     doubleCompare(ft0, ft1, .jumpTarget)
1652     dispatch(4)
1653
1654 .op2NotInt:
1655     ci2d t0, ft0
1656     btqz t1, tagTypeNumber, .slow
1657     addq tagTypeNumber, t1
1658     fq2d t1, ft1
1659     doubleCompare(ft0, ft1, .jumpTarget)
1660     dispatch(4)
1661
1662 .jumpTarget:
1663     dispatchIntIndirect(3)
1664
1665 .slow:
1666     callSlowPath(slowPath)
1667     dispatch(0)
1668 end
1669
1670
1671 _llint_op_switch_imm:
1672     traceExecution()
1673     loadisFromInstruction(3, t2)
1674     loadisFromInstruction(1, t3)
1675     loadConstantOrVariable(t2, t1)
1676     loadp CodeBlock[cfr], t2
1677     loadp CodeBlock::m_rareData[t2], t2
1678     muli sizeof SimpleJumpTable, t3    # FIXME: would be nice to peephole this!
1679     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1680     addp t3, t2
1681     bqb t1, tagTypeNumber, .opSwitchImmNotInt
1682     subi SimpleJumpTable::min[t2], t1
1683     biaeq t1, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchImmFallThrough
1684     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t3
1685     loadis [t3, t1, 4], t1
1686     btiz t1, .opSwitchImmFallThrough
1687     dispatch(t1)
1688
1689 .opSwitchImmNotInt:
1690     btqnz t1, tagTypeNumber, .opSwitchImmSlow   # Go slow if it's a double.
1691 .opSwitchImmFallThrough:
1692     dispatchIntIndirect(2)
1693
1694 .opSwitchImmSlow:
1695     callSlowPath(_llint_slow_path_switch_imm)
1696     dispatch(0)
1697
1698
1699 _llint_op_switch_char:
1700     traceExecution()
1701     loadisFromInstruction(3, t2)
1702     loadisFromInstruction(1, t3)
1703     loadConstantOrVariable(t2, t1)
1704     loadp CodeBlock[cfr], t2
1705     loadp CodeBlock::m_rareData[t2], t2
1706     muli sizeof SimpleJumpTable, t3
1707     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1708     addp t3, t2
1709     btqnz t1, tagMask, .opSwitchCharFallThrough
1710     bbneq JSCell::m_type[t1], StringType, .opSwitchCharFallThrough
1711     bineq JSString::m_length[t1], 1, .opSwitchCharFallThrough
1712     loadp JSString::m_value[t1], t0
1713     btpz  t0, .opSwitchOnRope
1714     loadp StringImpl::m_data8[t0], t1
1715     btinz StringImpl::m_hashAndFlags[t0], HashFlags8BitBuffer, .opSwitchChar8Bit
1716     loadh [t1], t0
1717     jmp .opSwitchCharReady
1718 .opSwitchChar8Bit:
1719     loadb [t1], t0
1720 .opSwitchCharReady:
1721     subi SimpleJumpTable::min[t2], t0
1722     biaeq t0, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchCharFallThrough
1723     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t2
1724     loadis [t2, t0, 4], t1
1725     btiz t1, .opSwitchCharFallThrough
1726     dispatch(t1)
1727
1728 .opSwitchCharFallThrough:
1729     dispatchIntIndirect(2)
1730
1731 .opSwitchOnRope:
1732     callSlowPath(_llint_slow_path_switch_char)
1733     dispatch(0)
1734
1735
1736 macro arrayProfileForCall()
1737     loadisFromInstruction(4, t3)
1738     negp t3
1739     loadq ThisArgumentOffset[cfr, t3, 8], t0
1740     btqnz t0, tagMask, .done
1741     loadpFromInstruction((CallOpCodeSize - 2), t1)
1742     loadi JSCell::m_structureID[t0], t3
1743     storei t3, ArrayProfile::m_lastSeenStructureID[t1]
1744 .done:
1745 end
1746
1747 macro doCall(slowPath, prepareCall)
1748     loadisFromInstruction(2, t0)
1749     loadpFromInstruction(5, t1)
1750     loadp LLIntCallLinkInfo::callee[t1], t2
1751     loadConstantOrVariable(t0, t3)
1752     bqneq t3, t2, .opCallSlow
1753     loadisFromInstruction(4, t3)
1754     lshifti 3, t3
1755     negp t3
1756     addp cfr, t3
1757     storeq t2, Callee[t3]
1758     loadisFromInstruction(3, t2)
1759     storei PC, ArgumentCount + TagOffset[cfr]
1760     storei t2, ArgumentCount + PayloadOffset[t3]
1761     move t3, sp
1762     prepareCall(LLIntCallLinkInfo::machineCodeTarget[t1], t2, t3, t4)
1763     callTargetFunction(LLIntCallLinkInfo::machineCodeTarget[t1])
1764
1765 .opCallSlow:
1766     slowPathForCall(slowPath, prepareCall)
1767 end
1768
1769 _llint_op_ret:
1770     traceExecution()
1771     checkSwitchToJITForEpilogue()
1772     loadisFromInstruction(1, t2)
1773     loadConstantOrVariable(t2, r0)
1774     doReturn()
1775
1776
1777 _llint_op_to_primitive:
1778     traceExecution()
1779     loadisFromInstruction(2, t2)
1780     loadisFromInstruction(1, t3)
1781     loadConstantOrVariable(t2, t0)
1782     btqnz t0, tagMask, .opToPrimitiveIsImm
1783     bbaeq JSCell::m_type[t0], ObjectType, .opToPrimitiveSlowCase
1784 .opToPrimitiveIsImm:
1785     storeq t0, [cfr, t3, 8]
1786     dispatch(3)
1787
1788 .opToPrimitiveSlowCase:
1789     callSlowPath(_slow_path_to_primitive)
1790     dispatch(3)
1791
1792
1793 _llint_op_catch:
1794     # This is where we end up from the JIT's throw trampoline (because the
1795     # machine code return address will be set to _llint_op_catch), and from
1796     # the interpreter's throw trampoline (see _llint_throw_trampoline).
1797     # The throwing code must have known that we were throwing to the interpreter,
1798     # and have set VM::targetInterpreterPCForThrow.
1799     loadp Callee[cfr], t3
1800     andp MarkedBlockMask, t3
1801     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1802     restoreCalleeSavesFromVMCalleeSavesBuffer(t3, t0)
1803     loadp VM::callFrameForCatch[t3], cfr
1804     storep 0, VM::callFrameForCatch[t3]
1805     restoreStackPointerAfterCall()
1806
1807     loadp CodeBlock[cfr], PB
1808     loadp CodeBlock::m_instructions[PB], PB
1809     loadp VM::targetInterpreterPCForThrow[t3], PC
1810     subp PB, PC
1811     rshiftp 3, PC
1812
1813     callSlowPath(_llint_slow_path_check_if_exception_is_uncatchable_and_notify_profiler)
1814     bpeq r1, 0, .isCatchableException
1815     jmp _llint_throw_from_slow_path_trampoline
1816
1817 .isCatchableException:
1818     loadp Callee[cfr], t3
1819     andp MarkedBlockMask, t3
1820     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1821
1822     loadq VM::m_exception[t3], t0
1823     storeq 0, VM::m_exception[t3]
1824     loadisFromInstruction(1, t2)
1825     storeq t0, [cfr, t2, 8]
1826
1827     loadq Exception::m_value[t0], t3
1828     loadisFromInstruction(2, t2)
1829     storeq t3, [cfr, t2, 8]
1830
1831     traceExecution()
1832     dispatch(3)
1833
1834
1835 _llint_op_end:
1836     traceExecution()
1837     checkSwitchToJITForEpilogue()
1838     loadisFromInstruction(1, t0)
1839     assertNotConstant(t0)
1840     loadq [cfr, t0, 8], r0
1841     doReturn()
1842
1843
1844 _llint_throw_from_slow_path_trampoline:
1845     loadp Callee[cfr], t1
1846     andp MarkedBlockMask, t1
1847     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1848     copyCalleeSavesToVMCalleeSavesBuffer(t1, t2)
1849
1850     callSlowPath(_llint_slow_path_handle_exception)
1851
1852     # When throwing from the interpreter (i.e. throwing from LLIntSlowPaths), so
1853     # the throw target is not necessarily interpreted code, we come to here.
1854     # This essentially emulates the JIT's throwing protocol.
1855     loadp Callee[cfr], t1
1856     andp MarkedBlockMask, t1
1857     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1858     jmp VM::targetMachinePCForThrow[t1]
1859
1860
1861 _llint_throw_during_call_trampoline:
1862     preserveReturnAddressAfterCall(t2)
1863     jmp _llint_throw_from_slow_path_trampoline
1864
1865
1866 macro nativeCallTrampoline(executableOffsetToFunction)
1867
1868     functionPrologue()
1869     storep 0, CodeBlock[cfr]
1870     loadp Callee[cfr], t0
1871     andp MarkedBlockMask, t0, t1
1872     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1873     storep cfr, VM::topCallFrame[t1]
1874     if ARM64 or C_LOOP
1875         storep lr, ReturnPC[cfr]
1876     end
1877     move cfr, a0
1878     loadp Callee[cfr], t1
1879     loadp JSFunction::m_executable[t1], t1
1880     checkStackPointerAlignment(t3, 0xdead0001)
1881     if C_LOOP
1882         cloopCallNative executableOffsetToFunction[t1]
1883     else
1884         if X86_64_WIN
1885             subp 32, sp
1886         end
1887         call executableOffsetToFunction[t1]
1888         if X86_64_WIN
1889             addp 32, sp
1890         end
1891     end
1892     loadp Callee[cfr], t3
1893     andp MarkedBlockMask, t3
1894     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1895
1896     functionEpilogue()
1897
1898     btqnz VM::m_exception[t3], .handleException
1899     ret
1900
1901 .handleException:
1902     storep cfr, VM::topCallFrame[t3]
1903     restoreStackPointerAfterCall()
1904     jmp _llint_throw_from_slow_path_trampoline
1905 end
1906
1907 macro getConstantScope(dst)
1908     loadpFromInstruction(6, t0)
1909     loadisFromInstruction(dst, t1)
1910     storeq t0, [cfr, t1, 8]
1911 end
1912
1913 macro varInjectionCheck(slowPath)
1914     loadp CodeBlock[cfr], t0
1915     loadp CodeBlock::m_globalObject[t0], t0
1916     loadp JSGlobalObject::m_varInjectionWatchpoint[t0], t0
1917     bbeq WatchpointSet::m_state[t0], IsInvalidated, slowPath
1918 end
1919
1920 macro resolveScope()
1921     loadisFromInstruction(5, t2)
1922     loadisFromInstruction(2, t0)
1923     loadp [cfr, t0, 8], t0
1924     btiz t2, .resolveScopeLoopEnd
1925
1926 .resolveScopeLoop:
1927     loadp JSScope::m_next[t0], t0
1928     subi 1, t2
1929     btinz t2, .resolveScopeLoop
1930
1931 .resolveScopeLoopEnd:
1932     loadisFromInstruction(1, t1)
1933     storeq t0, [cfr, t1, 8]
1934 end
1935
1936
1937 _llint_op_resolve_scope:
1938     traceExecution()
1939     loadisFromInstruction(4, t0)
1940
1941 #rGlobalProperty:
1942     bineq t0, GlobalProperty, .rGlobalVar
1943     getConstantScope(1)
1944     dispatch(7)
1945
1946 .rGlobalVar:
1947     bineq t0, GlobalVar, .rGlobalLexicalVar
1948     getConstantScope(1)
1949     dispatch(7)
1950
1951 .rGlobalLexicalVar:
1952     bineq t0, GlobalLexicalVar, .rClosureVar
1953     getConstantScope(1)
1954     dispatch(7)
1955
1956 .rClosureVar:
1957     bineq t0, ClosureVar, .rModuleVar
1958     resolveScope()
1959     dispatch(7)
1960
1961 .rModuleVar:
1962     bineq t0, ModuleVar, .rGlobalPropertyWithVarInjectionChecks
1963     getConstantScope(1)
1964     dispatch(7)
1965
1966 .rGlobalPropertyWithVarInjectionChecks:
1967     bineq t0, GlobalPropertyWithVarInjectionChecks, .rGlobalVarWithVarInjectionChecks
1968     varInjectionCheck(.rDynamic)
1969     getConstantScope(1)
1970     dispatch(7)
1971
1972 .rGlobalVarWithVarInjectionChecks:
1973     bineq t0, GlobalVarWithVarInjectionChecks, .rGlobalLexicalVarWithVarInjectionChecks
1974     varInjectionCheck(.rDynamic)
1975     getConstantScope(1)
1976     dispatch(7)
1977
1978 .rGlobalLexicalVarWithVarInjectionChecks:
1979     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .rClosureVarWithVarInjectionChecks
1980     varInjectionCheck(.rDynamic)
1981     getConstantScope(1)
1982     dispatch(7)
1983
1984 .rClosureVarWithVarInjectionChecks:
1985     bineq t0, ClosureVarWithVarInjectionChecks, .rDynamic
1986     varInjectionCheck(.rDynamic)
1987     resolveScope()
1988     dispatch(7)
1989
1990 .rDynamic:
1991     callSlowPath(_slow_path_resolve_scope)
1992     dispatch(7)
1993
1994
1995 macro loadWithStructureCheck(operand, slowPath)
1996     loadisFromInstruction(operand, t0)
1997     loadq [cfr, t0, 8], t0
1998     loadStructureWithScratch(t0, t2, t1)
1999     loadpFromInstruction(5, t1)
2000     bpneq t2, t1, slowPath
2001 end
2002
2003 macro getProperty(slow)
2004     loadisFromInstruction(6, t1)
2005     loadPropertyAtVariableOffset(t1, t0, t2, slow)
2006     valueProfile(t2, 7, t0)
2007     loadisFromInstruction(1, t0)
2008     storeq t2, [cfr, t0, 8]
2009 end
2010
2011 macro getGlobalVar(tdzCheckIfNecessary)
2012     loadpFromInstruction(6, t0)
2013     loadq [t0], t0
2014     tdzCheckIfNecessary(t0)
2015     valueProfile(t0, 7, t1)
2016     loadisFromInstruction(1, t1)
2017     storeq t0, [cfr, t1, 8]
2018 end
2019
2020 macro getClosureVar()
2021     loadisFromInstruction(6, t1)
2022     loadq JSEnvironmentRecord_variables[t0, t1, 8], t0
2023     valueProfile(t0, 7, t1)
2024     loadisFromInstruction(1, t1)
2025     storeq t0, [cfr, t1, 8]
2026 end
2027
2028 _llint_op_get_from_scope:
2029     traceExecution()
2030     loadisFromInstruction(4, t0)
2031     andi ResolveTypeMask, t0
2032
2033 #gGlobalProperty:
2034     bineq t0, GlobalProperty, .gGlobalVar
2035     loadWithStructureCheck(2, .gDynamic)
2036     getProperty(.gDynamic)
2037     dispatch(8)
2038
2039 .gGlobalVar:
2040     bineq t0, GlobalVar, .gGlobalLexicalVar
2041     getGlobalVar(macro(v) end)
2042     dispatch(8)
2043
2044 .gGlobalLexicalVar:
2045     bineq t0, GlobalLexicalVar, .gClosureVar
2046     getGlobalVar(
2047         macro (value)
2048             bqeq value, ValueEmpty, .gDynamic
2049         end)
2050     dispatch(8)
2051
2052 .gClosureVar:
2053     bineq t0, ClosureVar, .gGlobalPropertyWithVarInjectionChecks
2054     loadVariable(2, t0)
2055     getClosureVar()
2056     dispatch(8)
2057
2058 .gGlobalPropertyWithVarInjectionChecks:
2059     bineq t0, GlobalPropertyWithVarInjectionChecks, .gGlobalVarWithVarInjectionChecks
2060     loadWithStructureCheck(2, .gDynamic)
2061     getProperty(.gDynamic)
2062     dispatch(8)
2063
2064 .gGlobalVarWithVarInjectionChecks:
2065     bineq t0, GlobalVarWithVarInjectionChecks, .gGlobalLexicalVarWithVarInjectionChecks
2066     varInjectionCheck(.gDynamic)
2067     getGlobalVar(macro(v) end)
2068     dispatch(8)
2069
2070 .gGlobalLexicalVarWithVarInjectionChecks:
2071     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .gClosureVarWithVarInjectionChecks
2072     varInjectionCheck(.gDynamic)
2073     getGlobalVar(
2074         macro (value)
2075             bqeq value, ValueEmpty, .gDynamic
2076         end)
2077     dispatch(8)
2078
2079 .gClosureVarWithVarInjectionChecks:
2080     bineq t0, ClosureVarWithVarInjectionChecks, .gDynamic
2081     varInjectionCheck(.gDynamic)
2082     loadVariable(2, t0)
2083     getClosureVar()
2084     dispatch(8)
2085
2086 .gDynamic:
2087     callSlowPath(_llint_slow_path_get_from_scope)
2088     dispatch(8)
2089
2090
2091 macro putProperty(slow)
2092     loadisFromInstruction(3, t1)
2093     loadConstantOrVariable(t1, t2)
2094     loadisFromInstruction(6, t1)
2095     storePropertyAtVariableOffset(t1, t0, t2, slow)
2096 end
2097
2098 macro putGlobalVariable()
2099     loadisFromInstruction(3, t0)
2100     loadConstantOrVariable(t0, t1)
2101     loadpFromInstruction(5, t2)
2102     loadpFromInstruction(6, t0)
2103     notifyWrite(t2, .pDynamic)
2104     storeq t1, [t0]
2105 end
2106
2107 macro putClosureVar()
2108     loadisFromInstruction(3, t1)
2109     loadConstantOrVariable(t1, t2)
2110     loadisFromInstruction(6, t1)
2111     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2112 end
2113
2114 macro putLocalClosureVar()
2115     loadisFromInstruction(3, t1)
2116     loadConstantOrVariable(t1, t2)
2117     loadpFromInstruction(5, t3)
2118     btpz t3, .noVariableWatchpointSet
2119     notifyWrite(t3, .pDynamic)
2120 .noVariableWatchpointSet:
2121     loadisFromInstruction(6, t1)
2122     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2123 end
2124
2125 macro checkTDZInGlobalPutToScopeIfNecessary()
2126     loadisFromInstruction(4, t0)
2127     andi InitializationModeMask, t0
2128     rshifti InitializationModeShift, t0
2129     bieq t0, Initialization, .noNeedForTDZCheck
2130     loadpFromInstruction(6, t0)
2131     loadq [t0], t0
2132     bqeq t0, ValueEmpty, .pDynamic
2133 .noNeedForTDZCheck:
2134 end
2135
2136
2137 _llint_op_put_to_scope:
2138     traceExecution()
2139     loadisFromInstruction(4, t0)
2140     andi ResolveTypeMask, t0
2141
2142 #pLocalClosureVar:
2143     bineq t0, LocalClosureVar, .pGlobalProperty
2144     writeBarrierOnOperands(1, 3)
2145     loadVariable(1, t0)
2146     putLocalClosureVar()
2147     dispatch(7)
2148
2149 .pGlobalProperty:
2150     bineq t0, GlobalProperty, .pGlobalVar
2151     writeBarrierOnOperands(1, 3)
2152     loadWithStructureCheck(1, .pDynamic)
2153     putProperty(.pDynamic)
2154     dispatch(7)
2155
2156 .pGlobalVar:
2157     bineq t0, GlobalVar, .pGlobalLexicalVar
2158     writeBarrierOnGlobalObject(3)
2159     putGlobalVariable()
2160     dispatch(7)
2161
2162 .pGlobalLexicalVar:
2163     bineq t0, GlobalLexicalVar, .pClosureVar
2164     writeBarrierOnGlobalLexicalEnvironment(3)
2165     checkTDZInGlobalPutToScopeIfNecessary()
2166     putGlobalVariable()
2167     dispatch(7)
2168
2169 .pClosureVar:
2170     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
2171     writeBarrierOnOperands(1, 3)
2172     loadVariable(1, t0)
2173     putClosureVar()
2174     dispatch(7)
2175
2176 .pGlobalPropertyWithVarInjectionChecks:
2177     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
2178     writeBarrierOnOperands(1, 3)
2179     loadWithStructureCheck(1, .pDynamic)
2180     putProperty(.pDynamic)
2181     dispatch(7)
2182
2183 .pGlobalVarWithVarInjectionChecks:
2184     bineq t0, GlobalVarWithVarInjectionChecks, .pGlobalLexicalVarWithVarInjectionChecks
2185     writeBarrierOnGlobalObject(3)
2186     varInjectionCheck(.pDynamic)
2187     putGlobalVariable()
2188     dispatch(7)
2189
2190 .pGlobalLexicalVarWithVarInjectionChecks:
2191     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
2192     writeBarrierOnGlobalLexicalEnvironment(3)
2193     varInjectionCheck(.pDynamic)
2194     checkTDZInGlobalPutToScopeIfNecessary()
2195     putGlobalVariable()
2196     dispatch(7)
2197
2198 .pClosureVarWithVarInjectionChecks:
2199     bineq t0, ClosureVarWithVarInjectionChecks, .pModuleVar
2200     writeBarrierOnOperands(1, 3)
2201     varInjectionCheck(.pDynamic)
2202     loadVariable(1, t0)
2203     putClosureVar()
2204     dispatch(7)
2205
2206 .pModuleVar:
2207     bineq t0, ModuleVar, .pDynamic
2208     callSlowPath(_slow_path_throw_strict_mode_readonly_property_write_error)
2209     dispatch(7)
2210
2211 .pDynamic:
2212     callSlowPath(_llint_slow_path_put_to_scope)
2213     dispatch(7)
2214
2215
2216 _llint_op_get_from_arguments:
2217     traceExecution()
2218     loadVariable(2, t0)
2219     loadi 24[PB, PC, 8], t1
2220     loadq DirectArguments_storage[t0, t1, 8], t0
2221     valueProfile(t0, 4, t1)
2222     loadisFromInstruction(1, t1)
2223     storeq t0, [cfr, t1, 8]
2224     dispatch(5)
2225
2226
2227 _llint_op_put_to_arguments:
2228     traceExecution()
2229     writeBarrierOnOperands(1, 3)
2230     loadVariable(1, t0)
2231     loadi 16[PB, PC, 8], t1
2232     loadisFromInstruction(3, t3)
2233     loadConstantOrVariable(t3, t2)
2234     storeq t2, DirectArguments_storage[t0, t1, 8]
2235     dispatch(4)
2236
2237
2238 _llint_op_get_parent_scope:
2239     traceExecution()
2240     loadVariable(2, t0)
2241     loadp JSScope::m_next[t0], t0
2242     loadisFromInstruction(1, t1)
2243     storeq t0, [cfr, t1, 8]
2244     dispatch(3)
2245
2246
2247 _llint_op_profile_type:
2248     traceExecution()
2249     loadp CodeBlock[cfr], t1
2250     loadp CodeBlock::m_vm[t1], t1
2251     # t1 is holding the pointer to the typeProfilerLog.
2252     loadp VM::m_typeProfilerLog[t1], t1
2253     # t2 is holding the pointer to the current log entry.
2254     loadp TypeProfilerLog::m_currentLogEntryPtr[t1], t2
2255
2256     # t0 is holding the JSValue argument.
2257     loadisFromInstruction(1, t3)
2258     loadConstantOrVariable(t3, t0)
2259
2260     bqeq t0, ValueEmpty, .opProfileTypeDone
2261     # Store the JSValue onto the log entry.
2262     storeq t0, TypeProfilerLog::LogEntry::value[t2]
2263     
2264     # Store the TypeLocation onto the log entry.
2265     loadpFromInstruction(2, t3)
2266     storep t3, TypeProfilerLog::LogEntry::location[t2]
2267
2268     btqz t0, tagMask, .opProfileTypeIsCell
2269     storei 0, TypeProfilerLog::LogEntry::structureID[t2]
2270     jmp .opProfileTypeSkipIsCell
2271 .opProfileTypeIsCell:
2272     loadi JSCell::m_structureID[t0], t3
2273     storei t3, TypeProfilerLog::LogEntry::structureID[t2]
2274 .opProfileTypeSkipIsCell:
2275     
2276     # Increment the current log entry.
2277     addp sizeof TypeProfilerLog::LogEntry, t2
2278     storep t2, TypeProfilerLog::m_currentLogEntryPtr[t1]
2279
2280     loadp TypeProfilerLog::m_logEndPtr[t1], t1
2281     bpneq t2, t1, .opProfileTypeDone
2282     callSlowPath(_slow_path_profile_type_clear_log)
2283
2284 .opProfileTypeDone:
2285     dispatch(6)
2286
2287 _llint_op_profile_control_flow:
2288     traceExecution()
2289     loadpFromInstruction(1, t0)
2290     addq 1, BasicBlockLocation::m_executionCount[t0]
2291     dispatch(2)
2292
2293
2294 _llint_op_load_arrowfunction_this:
2295     traceExecution()
2296     loadp Callee[cfr], t0
2297     loadp JSArrowFunction::m_boundThis[t0], t0
2298     loadisFromInstruction(1, t1)
2299     storeq t0, [cfr, t1, 8]
2300     dispatch(2)
2301
2302
2303 _llint_op_get_rest_length:
2304     traceExecution()
2305     loadi PayloadOffset + ArgumentCount[cfr], t0
2306     subi 1, t0
2307     loadisFromInstruction(2, t1)
2308     bilteq t0, t1, .storeZero
2309     subi t1, t0
2310     jmp .boxUp
2311 .storeZero:
2312     move 0, t0
2313 .boxUp:
2314     orq tagTypeNumber, t0
2315     loadisFromInstruction(1, t1)
2316     storeq t0, [cfr, t1, 8]
2317     dispatch(3)