46e5616bf435e2361164d4493dc1256c38c34c22
[WebKit-https.git] / Source / JavaScriptCore / llint / LowLevelInterpreter64.asm
1 # Copyright (C) 2011-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1. Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2. Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
13 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
14 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
16 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
17 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
18 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
19 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
20 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
22 # THE POSSIBILITY OF SUCH DAMAGE.
23
24
25 # Utilities.
26 macro jumpToInstruction()
27     jmp [PB, PC, 8]
28 end
29
30 macro dispatch(advance)
31     addp advance, PC
32     jumpToInstruction()
33 end
34
35 macro dispatchInt(advance)
36     addi advance, PC
37     jumpToInstruction()
38 end
39
40 macro dispatchIntIndirect(offset)
41     dispatchInt(offset * 8[PB, PC, 8])
42 end
43
44 macro dispatchAfterCall()
45     loadi ArgumentCount + TagOffset[cfr], PC
46     loadp CodeBlock[cfr], PB
47     loadp CodeBlock::m_instructions[PB], PB
48     loadisFromInstruction(1, t1)
49     storeq r0, [cfr, t1, 8]
50     valueProfile(r0, (CallOpCodeSize - 1), t3)
51     dispatch(CallOpCodeSize)
52 end
53
54 macro cCall2(function)
55     checkStackPointerAlignment(t4, 0xbad0c002)
56     if X86_64 or ARM64
57         call function
58     elsif X86_64_WIN
59         # Note: this implementation is only correct if the return type size is > 8 bytes.
60         # See macro cCall2Void for an implementation when the return type <= 8 bytes.
61         # On Win64, when the return type is larger than 8 bytes, we need to allocate space on the stack for the return value.
62         # On entry rcx (a0), should contain a pointer to this stack space. The other parameters are shifted to the right,
63         # rdx (a1) should contain the first argument, and r8 (a2) should contain the second argument.
64         # On return, rax contains a pointer to this stack value, and we then need to copy the 16 byte return value into rax (r0) and rdx (r1)
65         # since the return value is expected to be split between the two.
66         # See http://msdn.microsoft.com/en-us/library/7572ztz4.aspx
67         move a1, a2
68         move a0, a1
69         subp 48, sp
70         move sp, a0
71         addp 32, a0
72         call function
73         addp 48, sp
74         move 8[r0], r1
75         move [r0], r0
76     elsif C_LOOP
77         cloopCallSlowPath function, a0, a1
78     else
79         error
80     end
81 end
82
83 macro cCall2Void(function)
84     if C_LOOP
85         cloopCallSlowPathVoid function, a0, a1
86     elsif X86_64_WIN
87         # Note: we cannot use the cCall2 macro for Win64 in this case,
88         # as the Win64 cCall2 implemenation is only correct when the return type size is > 8 bytes.
89         # On Win64, rcx and rdx are used for passing the first two parameters.
90         # We also need to make room on the stack for all four parameter registers.
91         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
92         subp 32, sp 
93         call function
94         addp 32, sp 
95     else
96         cCall2(function)
97     end
98 end
99
100 # This barely works. arg3 and arg4 should probably be immediates.
101 macro cCall4(function)
102     checkStackPointerAlignment(t4, 0xbad0c004)
103     if X86_64 or ARM64
104         call function
105     elsif X86_64_WIN
106         # On Win64, rcx, rdx, r8, and r9 are used for passing the first four parameters.
107         # We also need to make room on the stack for all four parameter registers.
108         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
109         subp 64, sp
110         call function
111         addp 64, sp
112     else
113         error
114     end
115 end
116
117 macro doVMEntry(makeCall)
118     functionPrologue()
119     pushCalleeSaves()
120
121     const entry = a0
122     const vm = a1
123     const protoCallFrame = a2
124
125     vmEntryRecord(cfr, sp)
126
127     checkStackPointerAlignment(t4, 0xbad0dc01)
128
129     storep vm, VMEntryRecord::m_vm[sp]
130     loadp VM::topCallFrame[vm], t4
131     storep t4, VMEntryRecord::m_prevTopCallFrame[sp]
132     loadp VM::topVMEntryFrame[vm], t4
133     storep t4, VMEntryRecord::m_prevTopVMEntryFrame[sp]
134
135     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t4
136     addp CallFrameHeaderSlots, t4, t4
137     lshiftp 3, t4
138     subp sp, t4, t3
139
140     # Ensure that we have enough additional stack capacity for the incoming args,
141     # and the frame for the JS code we're executing. We need to do this check
142     # before we start copying the args from the protoCallFrame below.
143     bpaeq t3, VM::m_jsStackLimit[vm], .stackHeightOK
144
145     if C_LOOP
146         move entry, t4
147         move vm, t5
148         cloopCallSlowPath _llint_stack_check_at_vm_entry, vm, t3
149         bpeq t0, 0, .stackCheckFailed
150         move t4, entry
151         move t5, vm
152         jmp .stackHeightOK
153
154 .stackCheckFailed:
155         move t4, entry
156         move t5, vm
157     end
158
159     move vm, a0
160     move protoCallFrame, a1
161     cCall2(_llint_throw_stack_overflow_error)
162
163     vmEntryRecord(cfr, t4)
164
165     loadp VMEntryRecord::m_vm[t4], vm
166     loadp VMEntryRecord::m_prevTopCallFrame[t4], extraTempReg
167     storep extraTempReg, VM::topCallFrame[vm]
168     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], extraTempReg
169     storep extraTempReg, VM::topVMEntryFrame[vm]
170
171     subp cfr, CalleeRegisterSaveSize, sp
172
173     popCalleeSaves()
174     functionEpilogue()
175     ret
176
177 .stackHeightOK:
178     move t3, sp
179     move 4, t3
180
181 .copyHeaderLoop:
182     subi 1, t3
183     loadq [protoCallFrame, t3, 8], extraTempReg
184     storeq extraTempReg, CodeBlock[sp, t3, 8]
185     btinz t3, .copyHeaderLoop
186
187     loadi PayloadOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], t4
188     subi 1, t4
189     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], extraTempReg
190     subi 1, extraTempReg
191
192     bieq t4, extraTempReg, .copyArgs
193     move ValueUndefined, t3
194 .fillExtraArgsLoop:
195     subi 1, extraTempReg
196     storeq t3, ThisArgumentOffset + 8[sp, extraTempReg, 8]
197     bineq t4, extraTempReg, .fillExtraArgsLoop
198
199 .copyArgs:
200     loadp ProtoCallFrame::args[protoCallFrame], t3
201
202 .copyArgsLoop:
203     btiz t4, .copyArgsDone
204     subi 1, t4
205     loadq [t3, t4, 8], extraTempReg
206     storeq extraTempReg, ThisArgumentOffset + 8[sp, t4, 8]
207     jmp .copyArgsLoop
208
209 .copyArgsDone:
210     if ARM64
211         move sp, t4
212         storep t4, VM::topCallFrame[vm]
213     else
214         storep sp, VM::topCallFrame[vm]
215     end
216     storep cfr, VM::topVMEntryFrame[vm]
217
218     checkStackPointerAlignment(extraTempReg, 0xbad0dc02)
219
220     makeCall(entry, t3)
221
222     # We may have just made a call into a JS function, so we can't rely on sp
223     # for anything but the fact that our own locals (ie the VMEntryRecord) are
224     # not below it. It also still has to be aligned, though.
225     checkStackPointerAlignment(t2, 0xbad0dc03)
226
227     vmEntryRecord(cfr, t4)
228
229     loadp VMEntryRecord::m_vm[t4], vm
230     loadp VMEntryRecord::m_prevTopCallFrame[t4], t2
231     storep t2, VM::topCallFrame[vm]
232     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], t2
233     storep t2, VM::topVMEntryFrame[vm]
234
235     subp cfr, CalleeRegisterSaveSize, sp
236
237     popCalleeSaves()
238     functionEpilogue()
239
240     ret
241 end
242
243
244 macro makeJavaScriptCall(entry, temp)
245     addp 16, sp
246     if C_LOOP
247         cloopCallJSFunction entry
248     else
249         call entry
250     end
251     subp 16, sp
252 end
253
254
255 macro makeHostFunctionCall(entry, temp)
256     move entry, temp
257     storep cfr, [sp]
258     move sp, a0
259     if C_LOOP
260         storep lr, 8[sp]
261         cloopCallNative temp
262     elsif X86_64_WIN
263         # We need to allocate 32 bytes on the stack for the shadow space.
264         subp 32, sp
265         call temp
266         addp 32, sp
267     else
268         call temp
269     end
270 end
271
272
273 _handleUncaughtException:
274     loadp Callee[cfr], t3
275     andp MarkedBlockMask, t3
276     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
277     restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(t3, t0)
278     loadp VM::callFrameForCatch[t3], cfr
279     storep 0, VM::callFrameForCatch[t3]
280
281     loadp CallerFrame[cfr], cfr
282     vmEntryRecord(cfr, t2)
283
284     loadp VMEntryRecord::m_vm[t2], t3
285     loadp VMEntryRecord::m_prevTopCallFrame[t2], extraTempReg
286     storep extraTempReg, VM::topCallFrame[t3]
287     loadp VMEntryRecord::m_prevTopVMEntryFrame[t2], extraTempReg
288     storep extraTempReg, VM::topVMEntryFrame[t3]
289
290     subp cfr, CalleeRegisterSaveSize, sp
291
292     popCalleeSaves()
293     functionEpilogue()
294     ret
295
296
297 macro prepareStateForCCall()
298     leap [PB, PC, 8], PC
299 end
300
301 macro restoreStateAfterCCall()
302     move r0, PC
303     subp PB, PC
304     rshiftp 3, PC
305 end
306
307 macro callSlowPath(slowPath)
308     prepareStateForCCall()
309     move cfr, a0
310     move PC, a1
311     cCall2(slowPath)
312     restoreStateAfterCCall()
313 end
314
315 macro traceOperand(fromWhere, operand)
316     prepareStateForCCall()
317     move fromWhere, a2
318     move operand, a3
319     move cfr, a0
320     move PC, a1
321     cCall4(_llint_trace_operand)
322     restoreStateAfterCCall()
323 end
324
325 macro traceValue(fromWhere, operand)
326     prepareStateForCCall()
327     move fromWhere, a2
328     move operand, a3
329     move cfr, a0
330     move PC, a1
331     cCall4(_llint_trace_value)
332     restoreStateAfterCCall()
333 end
334
335 # Call a slow path for call call opcodes.
336 macro callCallSlowPath(slowPath, action)
337     storei PC, ArgumentCount + TagOffset[cfr]
338     prepareStateForCCall()
339     move cfr, a0
340     move PC, a1
341     cCall2(slowPath)
342     action(r0, r1)
343 end
344
345 macro callWatchdogTimerHandler(throwHandler)
346     storei PC, ArgumentCount + TagOffset[cfr]
347     prepareStateForCCall()
348     move cfr, a0
349     move PC, a1
350     cCall2(_llint_slow_path_handle_watchdog_timer)
351     btpnz r0, throwHandler
352     loadi ArgumentCount + TagOffset[cfr], PC
353 end
354
355 macro checkSwitchToJITForLoop()
356     checkSwitchToJIT(
357         1,
358         macro()
359             storei PC, ArgumentCount + TagOffset[cfr]
360             prepareStateForCCall()
361             move cfr, a0
362             move PC, a1
363             cCall2(_llint_loop_osr)
364             btpz r0, .recover
365             move r1, sp
366             jmp r0
367         .recover:
368             loadi ArgumentCount + TagOffset[cfr], PC
369         end)
370 end
371
372 macro loadVariable(operand, value)
373     loadisFromInstruction(operand, value)
374     loadq [cfr, value, 8], value
375 end
376
377 # Index and value must be different registers. Index may be clobbered.
378 macro loadConstantOrVariable(index, value)
379     bpgteq index, FirstConstantRegisterIndex, .constant
380     loadq [cfr, index, 8], value
381     jmp .done
382 .constant:
383     loadp CodeBlock[cfr], value
384     loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value
385     subp FirstConstantRegisterIndex, index
386     loadq [value, index, 8], value
387 .done:
388 end
389
390 macro loadConstantOrVariableInt32(index, value, slow)
391     loadConstantOrVariable(index, value)
392     bqb value, tagTypeNumber, slow
393 end
394
395 macro loadConstantOrVariableCell(index, value, slow)
396     loadConstantOrVariable(index, value)
397     btqnz value, tagMask, slow
398 end
399
400 macro writeBarrierOnOperand(cellOperand)
401     loadisFromInstruction(cellOperand, t1)
402     loadConstantOrVariableCell(t1, t2, .writeBarrierDone)
403     skipIfIsRememberedOrInEden(t2, t1, t3, 
404         macro(cellState)
405             btbnz cellState, .writeBarrierDone
406             push PB, PC
407             move t2, a1 # t2 can be a0 (not on 64 bits, but better safe than sorry)
408             move cfr, a0
409             cCall2Void(_llint_write_barrier_slow)
410             pop PC, PB
411         end
412     )
413 .writeBarrierDone:
414 end
415
416 macro writeBarrierOnOperands(cellOperand, valueOperand)
417     loadisFromInstruction(valueOperand, t1)
418     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
419     btpz t0, .writeBarrierDone
420
421     writeBarrierOnOperand(cellOperand)
422 .writeBarrierDone:
423 end
424
425 macro writeBarrierOnGlobal(valueOperand, loadHelper)
426     loadisFromInstruction(valueOperand, t1)
427     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
428     btpz t0, .writeBarrierDone
429
430     loadHelper(t3)
431     skipIfIsRememberedOrInEden(t3, t1, t2,
432         macro(gcData)
433             btbnz gcData, .writeBarrierDone
434             push PB, PC
435             move cfr, a0
436             move t3, a1
437             cCall2Void(_llint_write_barrier_slow)
438             pop PC, PB
439         end
440     )
441 .writeBarrierDone:
442 end
443
444 macro writeBarrierOnGlobalObject(valueOperand)
445     writeBarrierOnGlobal(valueOperand,
446         macro(registerToStoreGlobal)
447             loadp CodeBlock[cfr], registerToStoreGlobal
448             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
449         end)
450 end
451
452 macro writeBarrierOnGlobalLexicalEnvironment(valueOperand)
453     writeBarrierOnGlobal(valueOperand,
454         macro(registerToStoreGlobal)
455             loadp CodeBlock[cfr], registerToStoreGlobal
456             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
457             loadp JSGlobalObject::m_globalLexicalEnvironment[registerToStoreGlobal], registerToStoreGlobal
458         end)
459 end
460
461 macro valueProfile(value, operand, scratch)
462     loadpFromInstruction(operand, scratch)
463     storeq value, ValueProfile::m_buckets[scratch]
464 end
465
466 macro structureIDToStructureWithScratch(structureIDThenStructure, scratch)
467     loadp CodeBlock[cfr], scratch
468     loadp CodeBlock::m_vm[scratch], scratch
469     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[scratch], scratch
470     loadp [scratch, structureIDThenStructure, 8], structureIDThenStructure
471 end
472
473 macro loadStructureWithScratch(cell, structure, scratch)
474     loadi JSCell::m_structureID[cell], structure
475     structureIDToStructureWithScratch(structure, scratch)
476 end
477
478 macro loadStructureAndClobberFirstArg(cell, structure)
479     loadi JSCell::m_structureID[cell], structure
480     loadp CodeBlock[cfr], cell
481     loadp CodeBlock::m_vm[cell], cell
482     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[cell], cell
483     loadp [cell, structure, 8], structure
484 end
485
486 macro storeStructureWithTypeInfo(cell, structure, scratch)
487     loadq Structure::m_blob + StructureIDBlob::u.doubleWord[structure], scratch
488     storeq scratch, JSCell::m_structureID[cell]
489 end
490
491 # Entrypoints into the interpreter.
492
493 # Expects that CodeBlock is in t1, which is what prologue() leaves behind.
494 macro functionArityCheck(doneLabel, slowPath)
495     loadi PayloadOffset + ArgumentCount[cfr], t0
496     biaeq t0, CodeBlock::m_numParameters[t1], doneLabel
497     prepareStateForCCall()
498     move cfr, a0
499     move PC, a1
500     cCall2(slowPath)   # This slowPath has the protocol: r0 = 0 => no error, r0 != 0 => error
501     btiz r0, .noError
502     move r1, cfr   # r1 contains caller frame
503     jmp _llint_throw_from_slow_path_trampoline
504
505 .noError:
506     loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[r1], t1
507     btiz t1, .continue
508     loadi PayloadOffset + ArgumentCount[cfr], t2
509     addi CallFrameHeaderSlots, t2
510
511     // Check if there are some unaligned slots we can use
512     move t1, t3
513     andi StackAlignmentSlots - 1, t3
514     btiz t3, .noExtraSlot
515     move ValueUndefined, t0
516 .fillExtraSlots:
517     storeq t0, [cfr, t2, 8]
518     addi 1, t2
519     bsubinz 1, t3, .fillExtraSlots
520     andi ~(StackAlignmentSlots - 1), t1
521     btiz t1, .continue
522
523 .noExtraSlot:
524     // Move frame up t1 slots
525     negq t1
526     move cfr, t3
527     subp CalleeSaveSpaceAsVirtualRegisters * 8, t3
528     addi CalleeSaveSpaceAsVirtualRegisters, t2
529 .copyLoop:
530     loadq [t3], t0
531     storeq t0, [t3, t1, 8]
532     addp 8, t3
533     bsubinz 1, t2, .copyLoop
534
535     // Fill new slots with JSUndefined
536     move t1, t2
537     move ValueUndefined, t0
538 .fillLoop:
539     storeq t0, [t3, t1, 8]
540     addp 8, t3
541     baddinz 1, t2, .fillLoop
542
543     lshiftp 3, t1
544     addp t1, cfr
545     addp t1, sp
546
547 .continue:
548     # Reload CodeBlock and reset PC, since the slow_path clobbered them.
549     loadp CodeBlock[cfr], t1
550     loadp CodeBlock::m_instructions[t1], PB
551     move 0, PC
552     jmp doneLabel
553 end
554
555 macro branchIfException(label)
556     loadp Callee[cfr], t3
557     andp MarkedBlockMask, t3
558     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
559     btqz VM::m_exception[t3], .noException
560     jmp label
561 .noException:
562 end
563
564
565 # Instruction implementations
566 _llint_op_enter:
567     traceExecution()
568     checkStackPointerAlignment(t2, 0xdead00e1)
569     loadp CodeBlock[cfr], t2                // t2<CodeBlock> = cfr.CodeBlock
570     loadi CodeBlock::m_numVars[t2], t2      // t2<size_t> = t2<CodeBlock>.m_numVars
571     subq CalleeSaveSpaceAsVirtualRegisters, t2
572     move cfr, t1
573     subq CalleeSaveSpaceAsVirtualRegisters * 8, t1
574     btiz t2, .opEnterDone
575     move ValueUndefined, t0
576     negi t2
577     sxi2q t2, t2
578 .opEnterLoop:
579     storeq t0, [t1, t2, 8]
580     addq 1, t2
581     btqnz t2, .opEnterLoop
582 .opEnterDone:
583     callSlowPath(_slow_path_enter)
584     dispatch(1)
585
586
587 _llint_op_get_scope:
588     traceExecution()
589     loadp Callee[cfr], t0
590     loadp JSCallee::m_scope[t0], t0
591     loadisFromInstruction(1, t1)
592     storeq t0, [cfr, t1, 8]
593     dispatch(2)
594
595
596 _llint_op_create_this:
597     traceExecution()
598     loadisFromInstruction(2, t0)
599     loadp [cfr, t0, 8], t0
600     bbneq JSCell::m_type[t0], JSFunctionType, .opCreateThisSlow
601     loadp JSFunction::m_rareData[t0], t3
602     btpz t3, .opCreateThisSlow
603     loadp FunctionRareData::m_objectAllocationProfile + ObjectAllocationProfile::m_allocator[t3], t1
604     loadp FunctionRareData::m_objectAllocationProfile + ObjectAllocationProfile::m_structure[t3], t2
605     btpz t1, .opCreateThisSlow
606     loadpFromInstruction(4, t3)
607     bpeq t3, 1, .hasSeenMultipleCallee
608     bpneq t3, t0, .opCreateThisSlow
609 .hasSeenMultipleCallee:
610     allocateJSObject(t1, t2, t0, t3, .opCreateThisSlow)
611     loadisFromInstruction(1, t1)
612     storeq t0, [cfr, t1, 8]
613     dispatch(5)
614
615 .opCreateThisSlow:
616     callSlowPath(_slow_path_create_this)
617     dispatch(5)
618
619
620 _llint_op_to_this:
621     traceExecution()
622     loadisFromInstruction(1, t0)
623     loadq [cfr, t0, 8], t0
624     btqnz t0, tagMask, .opToThisSlow
625     bbneq JSCell::m_type[t0], FinalObjectType, .opToThisSlow
626     loadStructureWithScratch(t0, t1, t2)
627     loadpFromInstruction(2, t2)
628     bpneq t1, t2, .opToThisSlow
629     dispatch(4)
630
631 .opToThisSlow:
632     callSlowPath(_slow_path_to_this)
633     dispatch(4)
634
635
636 _llint_op_new_object:
637     traceExecution()
638     loadpFromInstruction(3, t0)
639     loadp ObjectAllocationProfile::m_allocator[t0], t1
640     loadp ObjectAllocationProfile::m_structure[t0], t2
641     allocateJSObject(t1, t2, t0, t3, .opNewObjectSlow)
642     loadisFromInstruction(1, t1)
643     storeq t0, [cfr, t1, 8]
644     dispatch(4)
645
646 .opNewObjectSlow:
647     callSlowPath(_llint_slow_path_new_object)
648     dispatch(4)
649
650
651 _llint_op_check_tdz:
652     traceExecution()
653     loadisFromInstruction(1, t0)
654     loadConstantOrVariable(t0, t1)
655     bqneq t1, ValueEmpty, .opNotTDZ
656     callSlowPath(_slow_path_throw_tdz_error)
657
658 .opNotTDZ:
659     dispatch(2)
660
661
662 _llint_op_mov:
663     traceExecution()
664     loadisFromInstruction(2, t1)
665     loadisFromInstruction(1, t0)
666     loadConstantOrVariable(t1, t2)
667     storeq t2, [cfr, t0, 8]
668     dispatch(3)
669
670
671 _llint_op_not:
672     traceExecution()
673     loadisFromInstruction(2, t0)
674     loadisFromInstruction(1, t1)
675     loadConstantOrVariable(t0, t2)
676     xorq ValueFalse, t2
677     btqnz t2, ~1, .opNotSlow
678     xorq ValueTrue, t2
679     storeq t2, [cfr, t1, 8]
680     dispatch(3)
681
682 .opNotSlow:
683     callSlowPath(_slow_path_not)
684     dispatch(3)
685
686
687 macro equalityComparison(integerComparison, slowPath)
688     traceExecution()
689     loadisFromInstruction(3, t0)
690     loadisFromInstruction(2, t2)
691     loadisFromInstruction(1, t3)
692     loadConstantOrVariableInt32(t0, t1, .slow)
693     loadConstantOrVariableInt32(t2, t0, .slow)
694     integerComparison(t0, t1, t0)
695     orq ValueFalse, t0
696     storeq t0, [cfr, t3, 8]
697     dispatch(4)
698
699 .slow:
700     callSlowPath(slowPath)
701     dispatch(4)
702 end
703
704 _llint_op_eq:
705     equalityComparison(
706         macro (left, right, result) cieq left, right, result end,
707         _slow_path_eq)
708
709
710 _llint_op_neq:
711     equalityComparison(
712         macro (left, right, result) cineq left, right, result end,
713         _slow_path_neq)
714
715
716 macro equalNullComparison()
717     loadisFromInstruction(2, t0)
718     loadq [cfr, t0, 8], t0
719     btqnz t0, tagMask, .immediate
720     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
721     move 0, t0
722     jmp .done
723 .masqueradesAsUndefined:
724     loadStructureWithScratch(t0, t2, t1)
725     loadp CodeBlock[cfr], t0
726     loadp CodeBlock::m_globalObject[t0], t0
727     cpeq Structure::m_globalObject[t2], t0, t0
728     jmp .done
729 .immediate:
730     andq ~TagBitUndefined, t0
731     cqeq t0, ValueNull, t0
732 .done:
733 end
734
735 _llint_op_eq_null:
736     traceExecution()
737     equalNullComparison()
738     loadisFromInstruction(1, t1)
739     orq ValueFalse, t0
740     storeq t0, [cfr, t1, 8]
741     dispatch(3)
742
743
744 _llint_op_neq_null:
745     traceExecution()
746     equalNullComparison()
747     loadisFromInstruction(1, t1)
748     xorq ValueTrue, t0
749     storeq t0, [cfr, t1, 8]
750     dispatch(3)
751
752
753 macro strictEq(equalityOperation, slowPath)
754     traceExecution()
755     loadisFromInstruction(3, t0)
756     loadisFromInstruction(2, t2)
757     loadConstantOrVariable(t0, t1)
758     loadConstantOrVariable(t2, t0)
759     move t0, t2
760     orq t1, t2
761     btqz t2, tagMask, .slow
762     bqaeq t0, tagTypeNumber, .leftOK
763     btqnz t0, tagTypeNumber, .slow
764 .leftOK:
765     bqaeq t1, tagTypeNumber, .rightOK
766     btqnz t1, tagTypeNumber, .slow
767 .rightOK:
768     equalityOperation(t0, t1, t0)
769     loadisFromInstruction(1, t1)
770     orq ValueFalse, t0
771     storeq t0, [cfr, t1, 8]
772     dispatch(4)
773
774 .slow:
775     callSlowPath(slowPath)
776     dispatch(4)
777 end
778
779 _llint_op_stricteq:
780     strictEq(
781         macro (left, right, result) cqeq left, right, result end,
782         _slow_path_stricteq)
783
784
785 _llint_op_nstricteq:
786     strictEq(
787         macro (left, right, result) cqneq left, right, result end,
788         _slow_path_nstricteq)
789
790
791 macro preOp(arithmeticOperation, slowPath)
792     traceExecution()
793     loadisFromInstruction(1, t0)
794     loadq [cfr, t0, 8], t1
795     bqb t1, tagTypeNumber, .slow
796     arithmeticOperation(t1, .slow)
797     orq tagTypeNumber, t1
798     storeq t1, [cfr, t0, 8]
799     dispatch(2)
800
801 .slow:
802     callSlowPath(slowPath)
803     dispatch(2)
804 end
805
806 _llint_op_inc:
807     preOp(
808         macro (value, slow) baddio 1, value, slow end,
809         _slow_path_inc)
810
811
812 _llint_op_dec:
813     preOp(
814         macro (value, slow) bsubio 1, value, slow end,
815         _slow_path_dec)
816
817
818 _llint_op_to_number:
819     traceExecution()
820     loadisFromInstruction(2, t0)
821     loadisFromInstruction(1, t1)
822     loadConstantOrVariable(t0, t2)
823     bqaeq t2, tagTypeNumber, .opToNumberIsImmediate
824     btqz t2, tagTypeNumber, .opToNumberSlow
825 .opToNumberIsImmediate:
826     storeq t2, [cfr, t1, 8]
827     dispatch(3)
828
829 .opToNumberSlow:
830     callSlowPath(_slow_path_to_number)
831     dispatch(3)
832
833
834 _llint_op_to_string:
835     traceExecution()
836     loadisFromInstruction(2, t1)
837     loadisFromInstruction(1, t2)
838     loadConstantOrVariable(t1, t0)
839     btqnz t0, tagMask, .opToStringSlow
840     bbneq JSCell::m_type[t0], StringType, .opToStringSlow
841 .opToStringIsString:
842     storeq t0, [cfr, t2, 8]
843     dispatch(3)
844
845 .opToStringSlow:
846     callSlowPath(_slow_path_to_string)
847     dispatch(3)
848
849
850 _llint_op_negate:
851     traceExecution()
852     loadisFromInstruction(2, t0)
853     loadisFromInstruction(1, t1)
854     loadConstantOrVariable(t0, t2)
855     bqb t2, tagTypeNumber, .opNegateNotInt
856     btiz t2, 0x7fffffff, .opNegateSlow
857     negi t2
858     orq tagTypeNumber, t2
859     storeq t2, [cfr, t1, 8]
860     dispatch(3)
861 .opNegateNotInt:
862     btqz t2, tagTypeNumber, .opNegateSlow
863     xorq 0x8000000000000000, t2
864     storeq t2, [cfr, t1, 8]
865     dispatch(3)
866
867 .opNegateSlow:
868     callSlowPath(_slow_path_negate)
869     dispatch(3)
870
871
872 macro binaryOpCustomStore(integerOperationAndStore, doubleOperation, slowPath)
873     loadisFromInstruction(3, t0)
874     loadisFromInstruction(2, t2)
875     loadConstantOrVariable(t0, t1)
876     loadConstantOrVariable(t2, t0)
877     bqb t0, tagTypeNumber, .op1NotInt
878     bqb t1, tagTypeNumber, .op2NotInt
879     loadisFromInstruction(1, t2)
880     integerOperationAndStore(t1, t0, .slow, t2)
881     dispatch(5)
882
883 .op1NotInt:
884     # First operand is definitely not an int, the second operand could be anything.
885     btqz t0, tagTypeNumber, .slow
886     bqaeq t1, tagTypeNumber, .op1NotIntOp2Int
887     btqz t1, tagTypeNumber, .slow
888     addq tagTypeNumber, t1
889     fq2d t1, ft1
890     jmp .op1NotIntReady
891 .op1NotIntOp2Int:
892     ci2d t1, ft1
893 .op1NotIntReady:
894     loadisFromInstruction(1, t2)
895     addq tagTypeNumber, t0
896     fq2d t0, ft0
897     doubleOperation(ft1, ft0)
898     fd2q ft0, t0
899     subq tagTypeNumber, t0
900     storeq t0, [cfr, t2, 8]
901     dispatch(5)
902
903 .op2NotInt:
904     # First operand is definitely an int, the second is definitely not.
905     loadisFromInstruction(1, t2)
906     btqz t1, tagTypeNumber, .slow
907     ci2d t0, ft0
908     addq tagTypeNumber, t1
909     fq2d t1, ft1
910     doubleOperation(ft1, ft0)
911     fd2q ft0, t0
912     subq tagTypeNumber, t0
913     storeq t0, [cfr, t2, 8]
914     dispatch(5)
915
916 .slow:
917     callSlowPath(slowPath)
918     dispatch(5)
919 end
920
921 macro binaryOp(integerOperation, doubleOperation, slowPath)
922     binaryOpCustomStore(
923         macro (left, right, slow, index)
924             integerOperation(left, right, slow)
925             orq tagTypeNumber, right
926             storeq right, [cfr, index, 8]
927         end,
928         doubleOperation, slowPath)
929 end
930
931 _llint_op_add:
932     traceExecution()
933     binaryOp(
934         macro (left, right, slow) baddio left, right, slow end,
935         macro (left, right) addd left, right end,
936         _slow_path_add)
937
938
939 _llint_op_mul:
940     traceExecution()
941     binaryOpCustomStore(
942         macro (left, right, slow, index)
943             # Assume t3 is scratchable.
944             move right, t3
945             bmulio left, t3, slow
946             btinz t3, .done
947             bilt left, 0, slow
948             bilt right, 0, slow
949         .done:
950             orq tagTypeNumber, t3
951             storeq t3, [cfr, index, 8]
952         end,
953         macro (left, right) muld left, right end,
954         _slow_path_mul)
955
956
957 _llint_op_sub:
958     traceExecution()
959     binaryOp(
960         macro (left, right, slow) bsubio left, right, slow end,
961         macro (left, right) subd left, right end,
962         _slow_path_sub)
963
964
965 _llint_op_div:
966     traceExecution()
967     if X86_64 or X86_64_WIN
968         binaryOpCustomStore(
969             macro (left, right, slow, index)
970                 # Assume t3 is scratchable.
971                 btiz left, slow
972                 bineq left, -1, .notNeg2TwoThe31DivByNeg1
973                 bieq right, -2147483648, .slow
974             .notNeg2TwoThe31DivByNeg1:
975                 btinz right, .intOK
976                 bilt left, 0, slow
977             .intOK:
978                 move left, t3
979                 move right, t0
980                 cdqi
981                 idivi t3
982                 btinz t1, slow
983                 orq tagTypeNumber, t0
984                 storeq t0, [cfr, index, 8]
985             end,
986             macro (left, right) divd left, right end,
987             _slow_path_div)
988     else
989         callSlowPath(_slow_path_div)
990         dispatch(5)
991     end
992
993
994 macro bitOp(operation, slowPath, advance)
995     loadisFromInstruction(3, t0)
996     loadisFromInstruction(2, t2)
997     loadisFromInstruction(1, t3)
998     loadConstantOrVariable(t0, t1)
999     loadConstantOrVariable(t2, t0)
1000     bqb t0, tagTypeNumber, .slow
1001     bqb t1, tagTypeNumber, .slow
1002     operation(t1, t0)
1003     orq tagTypeNumber, t0
1004     storeq t0, [cfr, t3, 8]
1005     dispatch(advance)
1006
1007 .slow:
1008     callSlowPath(slowPath)
1009     dispatch(advance)
1010 end
1011
1012 _llint_op_lshift:
1013     traceExecution()
1014     bitOp(
1015         macro (left, right) lshifti left, right end,
1016         _slow_path_lshift,
1017         4)
1018
1019
1020 _llint_op_rshift:
1021     traceExecution()
1022     bitOp(
1023         macro (left, right) rshifti left, right end,
1024         _slow_path_rshift,
1025         4)
1026
1027
1028 _llint_op_urshift:
1029     traceExecution()
1030     bitOp(
1031         macro (left, right) urshifti left, right end,
1032         _slow_path_urshift,
1033         4)
1034
1035
1036 _llint_op_unsigned:
1037     traceExecution()
1038     loadisFromInstruction(1, t0)
1039     loadisFromInstruction(2, t1)
1040     loadConstantOrVariable(t1, t2)
1041     bilt t2, 0, .opUnsignedSlow
1042     storeq t2, [cfr, t0, 8]
1043     dispatch(3)
1044 .opUnsignedSlow:
1045     callSlowPath(_slow_path_unsigned)
1046     dispatch(3)
1047
1048
1049 _llint_op_bitand:
1050     traceExecution()
1051     bitOp(
1052         macro (left, right) andi left, right end,
1053         _slow_path_bitand,
1054         5)
1055
1056
1057 _llint_op_bitxor:
1058     traceExecution()
1059     bitOp(
1060         macro (left, right) xori left, right end,
1061         _slow_path_bitxor,
1062         5)
1063
1064
1065 _llint_op_bitor:
1066     traceExecution()
1067     bitOp(
1068         macro (left, right) ori left, right end,
1069         _slow_path_bitor,
1070         5)
1071
1072
1073 _llint_op_overrides_has_instance:
1074     traceExecution()
1075     loadisFromInstruction(1, t3)
1076
1077     loadisFromInstruction(3, t1)
1078     loadConstantOrVariable(t1, t0)
1079     loadp CodeBlock[cfr], t2
1080     loadp CodeBlock::m_globalObject[t2], t2
1081     loadp JSGlobalObject::m_functionProtoHasInstanceSymbolFunction[t2], t2
1082     bqneq t0, t2, .opOverridesHasInstanceNotDefaultSymbol
1083
1084     loadisFromInstruction(2, t1)
1085     loadConstantOrVariable(t1, t0)
1086     tbz JSCell::m_flags[t0], ImplementsDefaultHasInstance, t1
1087     orq ValueFalse, t1
1088     storeq t1, [cfr, t3, 8]
1089     dispatch(4)
1090
1091 .opOverridesHasInstanceNotDefaultSymbol:
1092     storeq ValueTrue, [cfr, t3, 8]
1093     dispatch(4)
1094
1095
1096 _llint_op_instanceof_custom:
1097     traceExecution()
1098     callSlowPath(_llint_slow_path_instanceof_custom)
1099     dispatch(5)
1100
1101
1102 _llint_op_is_empty:
1103     traceExecution()
1104     loadisFromInstruction(2, t1)
1105     loadisFromInstruction(1, t2)
1106     loadConstantOrVariable(t1, t0)
1107     cqeq t0, ValueEmpty, t3
1108     orq ValueFalse, t3
1109     storeq t3, [cfr, t2, 8]
1110     dispatch(3)
1111
1112
1113 _llint_op_is_undefined:
1114     traceExecution()
1115     loadisFromInstruction(2, t1)
1116     loadisFromInstruction(1, t2)
1117     loadConstantOrVariable(t1, t0)
1118     btqz t0, tagMask, .opIsUndefinedCell
1119     cqeq t0, ValueUndefined, t3
1120     orq ValueFalse, t3
1121     storeq t3, [cfr, t2, 8]
1122     dispatch(3)
1123 .opIsUndefinedCell:
1124     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
1125     move ValueFalse, t1
1126     storeq t1, [cfr, t2, 8]
1127     dispatch(3)
1128 .masqueradesAsUndefined:
1129     loadStructureWithScratch(t0, t3, t1)
1130     loadp CodeBlock[cfr], t1
1131     loadp CodeBlock::m_globalObject[t1], t1
1132     cpeq Structure::m_globalObject[t3], t1, t0
1133     orq ValueFalse, t0
1134     storeq t0, [cfr, t2, 8]
1135     dispatch(3)
1136
1137
1138 _llint_op_is_boolean:
1139     traceExecution()
1140     loadisFromInstruction(2, t1)
1141     loadisFromInstruction(1, t2)
1142     loadConstantOrVariable(t1, t0)
1143     xorq ValueFalse, t0
1144     tqz t0, ~1, t0
1145     orq ValueFalse, t0
1146     storeq t0, [cfr, t2, 8]
1147     dispatch(3)
1148
1149
1150 _llint_op_is_number:
1151     traceExecution()
1152     loadisFromInstruction(2, t1)
1153     loadisFromInstruction(1, t2)
1154     loadConstantOrVariable(t1, t0)
1155     tqnz t0, tagTypeNumber, t1
1156     orq ValueFalse, t1
1157     storeq t1, [cfr, t2, 8]
1158     dispatch(3)
1159
1160
1161 _llint_op_is_string:
1162     traceExecution()
1163     loadisFromInstruction(2, t1)
1164     loadisFromInstruction(1, t2)
1165     loadConstantOrVariable(t1, t0)
1166     btqnz t0, tagMask, .opIsStringNotCell
1167     cbeq JSCell::m_type[t0], StringType, t1
1168     orq ValueFalse, t1
1169     storeq t1, [cfr, t2, 8]
1170     dispatch(3)
1171 .opIsStringNotCell:
1172     storeq ValueFalse, [cfr, t2, 8]
1173     dispatch(3)
1174
1175
1176 _llint_op_is_object:
1177     traceExecution()
1178     loadisFromInstruction(2, t1)
1179     loadisFromInstruction(1, t2)
1180     loadConstantOrVariable(t1, t0)
1181     btqnz t0, tagMask, .opIsObjectNotCell
1182     cbaeq JSCell::m_type[t0], ObjectType, t1
1183     orq ValueFalse, t1
1184     storeq t1, [cfr, t2, 8]
1185     dispatch(3)
1186 .opIsObjectNotCell:
1187     storeq ValueFalse, [cfr, t2, 8]
1188     dispatch(3)
1189
1190
1191 macro loadPropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1192     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1193     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1194     negi propertyOffsetAsInt
1195     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1196     jmp .ready
1197 .isInline:
1198     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1199 .ready:
1200     loadq (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8], value
1201 end
1202
1203
1204 macro storePropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1205     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1206     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1207     negi propertyOffsetAsInt
1208     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1209     jmp .ready
1210 .isInline:
1211     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1212 .ready:
1213     storeq value, (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8]
1214 end
1215
1216 _llint_op_get_by_id:
1217     traceExecution()
1218     loadisFromInstruction(2, t0)
1219     loadConstantOrVariableCell(t0, t3, .opGetByIdSlow)
1220     loadi JSCell::m_structureID[t3], t1
1221     loadisFromInstruction(4, t2)
1222     bineq t2, t1, .opGetByIdSlow
1223     loadisFromInstruction(5, t1)
1224     loadisFromInstruction(1, t2)
1225     loadPropertyAtVariableOffset(t1, t3, t0)
1226     storeq t0, [cfr, t2, 8]
1227     valueProfile(t0, 8, t1)
1228     dispatch(9)
1229
1230 .opGetByIdSlow:
1231     callSlowPath(_llint_slow_path_get_by_id)
1232     dispatch(9)
1233
1234
1235 _llint_op_get_array_length:
1236     traceExecution()
1237     loadisFromInstruction(2, t0)
1238     loadpFromInstruction(4, t1)
1239     loadConstantOrVariableCell(t0, t3, .opGetArrayLengthSlow)
1240     move t3, t2
1241     arrayProfile(t2, t1, t0)
1242     btiz t2, IsArray, .opGetArrayLengthSlow
1243     btiz t2, IndexingShapeMask, .opGetArrayLengthSlow
1244     loadisFromInstruction(1, t1)
1245     loadp JSObject::m_butterfly[t3], t0
1246     loadi -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], t0
1247     bilt t0, 0, .opGetArrayLengthSlow
1248     orq tagTypeNumber, t0
1249     valueProfile(t0, 8, t2)
1250     storeq t0, [cfr, t1, 8]
1251     dispatch(9)
1252
1253 .opGetArrayLengthSlow:
1254     callSlowPath(_llint_slow_path_get_by_id)
1255     dispatch(9)
1256
1257
1258 _llint_op_put_by_id:
1259     traceExecution()
1260     writeBarrierOnOperands(1, 3)
1261     loadisFromInstruction(1, t3)
1262     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
1263     loadisFromInstruction(4, t2)
1264     bineq t2, JSCell::m_structureID[t0], .opPutByIdSlow
1265
1266     # At this point, we have:
1267     # t2 -> current structure ID
1268     # t0 -> object base
1269
1270     loadisFromInstruction(3, t1)
1271     loadConstantOrVariable(t1, t3)
1272
1273     loadpFromInstruction(8, t1)
1274
1275     # At this point, we have:
1276     # t0 -> object base
1277     # t1 -> put by id flags
1278     # t2 -> current structure ID
1279     # t3 -> value to put
1280
1281     btpnz t1, PutByIdPrimaryTypeMask, .opPutByIdTypeCheckObjectWithStructureOrOther
1282
1283     # We have one of the non-structure type checks. Find out which one.
1284     andp PutByIdSecondaryTypeMask, t1
1285     bplt t1, PutByIdSecondaryTypeString, .opPutByIdTypeCheckLessThanString
1286
1287     # We are one of the following: String, Symbol, Object, ObjectOrOther, Top
1288     bplt t1, PutByIdSecondaryTypeObjectOrOther, .opPutByIdTypeCheckLessThanObjectOrOther
1289
1290     # We are either ObjectOrOther or Top.
1291     bpeq t1, PutByIdSecondaryTypeTop, .opPutByIdDoneCheckingTypes
1292
1293     # Check if we are ObjectOrOther.
1294     btqz t3, tagMask, .opPutByIdTypeCheckObject
1295 .opPutByIdTypeCheckOther:
1296     andq ~TagBitUndefined, t3
1297     bqeq t3, ValueNull, .opPutByIdDoneCheckingTypes
1298     jmp .opPutByIdSlow
1299
1300 .opPutByIdTypeCheckLessThanObjectOrOther:
1301     # We are either String, Symbol or Object.
1302     btqnz t3, tagMask, .opPutByIdSlow
1303     bpeq t1, PutByIdSecondaryTypeObject, .opPutByIdTypeCheckObject
1304     bpeq t1, PutByIdSecondaryTypeSymbol, .opPutByIdTypeCheckSymbol
1305     bbeq JSCell::m_type[t3], StringType, .opPutByIdDoneCheckingTypes
1306     jmp .opPutByIdSlow
1307 .opPutByIdTypeCheckObject:
1308     bbaeq JSCell::m_type[t3], ObjectType, .opPutByIdDoneCheckingTypes
1309     jmp .opPutByIdSlow
1310 .opPutByIdTypeCheckSymbol:
1311     bbeq JSCell::m_type[t3], SymbolType, .opPutByIdDoneCheckingTypes
1312     jmp .opPutByIdSlow
1313
1314 .opPutByIdTypeCheckLessThanString:
1315     # We are one of the following: Bottom, Boolean, Other, Int32, Number
1316     bplt t1, PutByIdSecondaryTypeInt32, .opPutByIdTypeCheckLessThanInt32
1317
1318     # We are either Int32 or Number.
1319     bpeq t1, PutByIdSecondaryTypeNumber, .opPutByIdTypeCheckNumber
1320
1321     bqaeq t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1322     jmp .opPutByIdSlow
1323
1324 .opPutByIdTypeCheckNumber:
1325     btqnz t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1326     jmp .opPutByIdSlow
1327
1328 .opPutByIdTypeCheckLessThanInt32:
1329     # We are one of the following: Bottom, Boolean, Other.
1330     bpneq t1, PutByIdSecondaryTypeBoolean, .opPutByIdTypeCheckBottomOrOther
1331     xorq ValueFalse, t3
1332     btqz t3, ~1, .opPutByIdDoneCheckingTypes
1333     jmp .opPutByIdSlow
1334
1335 .opPutByIdTypeCheckBottomOrOther:
1336     bpeq t1, PutByIdSecondaryTypeOther, .opPutByIdTypeCheckOther
1337     jmp .opPutByIdSlow
1338
1339 .opPutByIdTypeCheckObjectWithStructureOrOther:
1340     btqz t3, tagMask, .opPutByIdTypeCheckObjectWithStructure
1341     btpnz t1, PutByIdPrimaryTypeObjectWithStructureOrOther, .opPutByIdTypeCheckOther
1342     jmp .opPutByIdSlow
1343
1344 .opPutByIdTypeCheckObjectWithStructure:
1345     urshiftp 3, t1
1346     bineq t1, JSCell::m_structureID[t3], .opPutByIdSlow
1347
1348 .opPutByIdDoneCheckingTypes:
1349     loadisFromInstruction(6, t1)
1350     
1351     btiz t1, .opPutByIdNotTransition
1352
1353     # This is the transition case. t1 holds the new structureID. t2 holds the old structure ID.
1354     # If we have a chain, we need to check it. t0 is the base. We may clobber t1 to use it as
1355     # scratch.
1356     loadpFromInstruction(7, t3)
1357     btpz t3, .opPutByIdTransitionDirect
1358
1359     loadp StructureChain::m_vector[t3], t3
1360     assert(macro (ok) btpnz t3, ok end)
1361
1362     structureIDToStructureWithScratch(t2, t1)
1363     loadq Structure::m_prototype[t2], t2
1364     bqeq t2, ValueNull, .opPutByIdTransitionChainDone
1365 .opPutByIdTransitionChainLoop:
1366     # At this point, t2 contains a prototye, and [t3] contains the Structure* that we want that
1367     # prototype to have. We don't want to have to load the Structure* for t2. Instead, we load
1368     # the Structure* from [t3], and then we compare its id to the id in the header of t2.
1369     loadp [t3], t1
1370     loadi JSCell::m_structureID[t2], t2
1371     # Now, t1 has the Structure* and t2 has the StructureID that we want that Structure* to have.
1372     bineq t2, Structure::m_blob + StructureIDBlob::u.fields.structureID[t1], .opPutByIdSlow
1373     addp 8, t3
1374     loadq Structure::m_prototype[t1], t2
1375     bqneq t2, ValueNull, .opPutByIdTransitionChainLoop
1376
1377 .opPutByIdTransitionChainDone:
1378     # Reload the new structure, since we clobbered it above.
1379     loadisFromInstruction(6, t1)
1380
1381 .opPutByIdTransitionDirect:
1382     storei t1, JSCell::m_structureID[t0]
1383
1384 .opPutByIdNotTransition:
1385     # The only thing live right now is t0, which holds the base.
1386     loadisFromInstruction(3, t1)
1387     loadConstantOrVariable(t1, t2)
1388     loadisFromInstruction(5, t1)
1389     storePropertyAtVariableOffset(t1, t0, t2)
1390     dispatch(9)
1391
1392 .opPutByIdSlow:
1393     callSlowPath(_llint_slow_path_put_by_id)
1394     dispatch(9)
1395
1396 macro finishGetByVal(result, scratch)
1397     loadisFromInstruction(1, scratch)
1398     storeq result, [cfr, scratch, 8]
1399     valueProfile(result, 5, scratch)
1400     dispatch(6)
1401 end
1402
1403 macro finishIntGetByVal(result, scratch)
1404     orq tagTypeNumber, result
1405     finishGetByVal(result, scratch)
1406 end
1407
1408 macro finishDoubleGetByVal(result, scratch1, scratch2)
1409     fd2q result, scratch1
1410     subq tagTypeNumber, scratch1
1411     finishGetByVal(scratch1, scratch2)
1412 end
1413
1414 _llint_op_get_by_val:
1415     traceExecution()
1416     loadisFromInstruction(2, t2)
1417     loadConstantOrVariableCell(t2, t0, .opGetByValSlow)
1418     loadpFromInstruction(4, t3)
1419     move t0, t2
1420     arrayProfile(t2, t3, t1)
1421     loadisFromInstruction(3, t3)
1422     loadConstantOrVariableInt32(t3, t1, .opGetByValSlow)
1423     sxi2q t1, t1
1424     loadp JSObject::m_butterfly[t0], t3
1425     andi IndexingShapeMask, t2
1426     bieq t2, Int32Shape, .opGetByValIsContiguous
1427     bineq t2, ContiguousShape, .opGetByValNotContiguous
1428 .opGetByValIsContiguous:
1429
1430     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1431     loadisFromInstruction(1, t0)
1432     loadq [t3, t1, 8], t2
1433     btqz t2, .opGetByValOutOfBounds
1434     jmp .opGetByValDone
1435
1436 .opGetByValNotContiguous:
1437     bineq t2, DoubleShape, .opGetByValNotDouble
1438     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1439     loadis 8[PB, PC, 8], t0
1440     loadd [t3, t1, 8], ft0
1441     bdnequn ft0, ft0, .opGetByValOutOfBounds
1442     fd2q ft0, t2
1443     subq tagTypeNumber, t2
1444     jmp .opGetByValDone
1445     
1446 .opGetByValNotDouble:
1447     subi ArrayStorageShape, t2
1448     bia t2, SlowPutArrayStorageShape - ArrayStorageShape, .opGetByValNotIndexedStorage
1449     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t3], .opGetByValOutOfBounds
1450     loadisFromInstruction(1, t0)
1451     loadq ArrayStorage::m_vector[t3, t1, 8], t2
1452     btqz t2, .opGetByValOutOfBounds
1453
1454 .opGetByValDone:
1455     storeq t2, [cfr, t0, 8]
1456     valueProfile(t2, 5, t0)
1457     dispatch(6)
1458
1459 .opGetByValOutOfBounds:
1460     loadpFromInstruction(4, t0)
1461     storeb 1, ArrayProfile::m_outOfBounds[t0]
1462
1463 .opGetByValNotIndexedStorage:
1464     # First lets check if we even have a typed array. This lets us do some boilerplate up front.
1465     loadb JSCell::m_type[t0], t2
1466     subi FirstArrayType, t2
1467     bia t2, LastArrayType - FirstArrayType, .opGetByValSlow
1468     
1469     # Sweet, now we know that we have a typed array. Do some basic things now.
1470     loadp JSArrayBufferView::m_vector[t0], t3
1471     biaeq t1, JSArrayBufferView::m_length[t0], .opGetByValSlow
1472     
1473     # Now bisect through the various types. Note that we can treat Uint8ArrayType and
1474     # Uint8ClampedArrayType the same.
1475     bia t2, Uint8ClampedArrayType - FirstArrayType, .opGetByValAboveUint8ClampedArray
1476     
1477     # We have one of Int8ArrayType .. Uint8ClampedArrayType.
1478     bia t2, Int16ArrayType - FirstArrayType, .opGetByValInt32ArrayOrUint8Array
1479     
1480     # We have one of Int8ArrayType or Int16ArrayType
1481     bineq t2, Int8ArrayType - FirstArrayType, .opGetByValInt16Array
1482     
1483     # We have Int8ArrayType
1484     loadbs [t3, t1], t0
1485     finishIntGetByVal(t0, t1)
1486
1487 .opGetByValInt16Array:
1488     loadhs [t3, t1, 2], t0
1489     finishIntGetByVal(t0, t1)
1490
1491 .opGetByValInt32ArrayOrUint8Array:
1492     # We have one of Int16Array, Uint8Array, or Uint8ClampedArray.
1493     bieq t2, Int32ArrayType - FirstArrayType, .opGetByValInt32Array
1494     
1495     # We have either Uint8Array or Uint8ClampedArray. They behave the same so that's cool.
1496     loadb [t3, t1], t0
1497     finishIntGetByVal(t0, t1)
1498
1499 .opGetByValInt32Array:
1500     loadi [t3, t1, 4], t0
1501     finishIntGetByVal(t0, t1)
1502
1503 .opGetByValAboveUint8ClampedArray:
1504     # We have one of Uint16ArrayType .. Float64ArrayType.
1505     bia t2, Uint32ArrayType - FirstArrayType, .opGetByValAboveUint32Array
1506     
1507     # We have either Uint16ArrayType or Uint32ArrayType.
1508     bieq t2, Uint32ArrayType - FirstArrayType, .opGetByValUint32Array
1509
1510     # We have Uint16ArrayType.
1511     loadh [t3, t1, 2], t0
1512     finishIntGetByVal(t0, t1)
1513
1514 .opGetByValUint32Array:
1515     # This is the hardest part because of large unsigned values.
1516     loadi [t3, t1, 4], t0
1517     bilt t0, 0, .opGetByValSlow # This case is still awkward to implement in LLInt.
1518     finishIntGetByVal(t0, t1)
1519
1520 .opGetByValAboveUint32Array:
1521     # We have one of Float32ArrayType or Float64ArrayType. Sadly, we cannot handle Float32Array
1522     # inline yet. That would require some offlineasm changes.
1523     bieq t2, Float32ArrayType - FirstArrayType, .opGetByValSlow
1524
1525     # We have Float64ArrayType.
1526     loadd [t3, t1, 8], ft0
1527     finishDoubleGetByVal(ft0, t0, t1)
1528
1529 .opGetByValSlow:
1530     callSlowPath(_llint_slow_path_get_by_val)
1531     dispatch(6)
1532
1533
1534 macro contiguousPutByVal(storeCallback)
1535     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .outOfBounds
1536 .storeResult:
1537     loadisFromInstruction(3, t2)
1538     storeCallback(t2, t1, [t0, t3, 8])
1539     dispatch(5)
1540
1541 .outOfBounds:
1542     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1543     loadp 32[PB, PC, 8], t2
1544     storeb 1, ArrayProfile::m_mayStoreToHole[t2]
1545     addi 1, t3, t2
1546     storei t2, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1547     jmp .storeResult
1548 end
1549
1550 macro putByVal(slowPath)
1551     traceExecution()
1552     writeBarrierOnOperands(1, 3)
1553     loadisFromInstruction(1, t0)
1554     loadConstantOrVariableCell(t0, t1, .opPutByValSlow)
1555     loadpFromInstruction(4, t3)
1556     move t1, t2
1557     arrayProfile(t2, t3, t0)
1558     loadisFromInstruction(2, t0)
1559     loadConstantOrVariableInt32(t0, t3, .opPutByValSlow)
1560     sxi2q t3, t3
1561     loadp JSObject::m_butterfly[t1], t0
1562     andi IndexingShapeMask, t2
1563     bineq t2, Int32Shape, .opPutByValNotInt32
1564     contiguousPutByVal(
1565         macro (operand, scratch, address)
1566             loadConstantOrVariable(operand, scratch)
1567             bpb scratch, tagTypeNumber, .opPutByValSlow
1568             storep scratch, address
1569         end)
1570
1571 .opPutByValNotInt32:
1572     bineq t2, DoubleShape, .opPutByValNotDouble
1573     contiguousPutByVal(
1574         macro (operand, scratch, address)
1575             loadConstantOrVariable(operand, scratch)
1576             bqb scratch, tagTypeNumber, .notInt
1577             ci2d scratch, ft0
1578             jmp .ready
1579         .notInt:
1580             addp tagTypeNumber, scratch
1581             fq2d scratch, ft0
1582             bdnequn ft0, ft0, .opPutByValSlow
1583         .ready:
1584             stored ft0, address
1585         end)
1586
1587 .opPutByValNotDouble:
1588     bineq t2, ContiguousShape, .opPutByValNotContiguous
1589     contiguousPutByVal(
1590         macro (operand, scratch, address)
1591             loadConstantOrVariable(operand, scratch)
1592             storep scratch, address
1593         end)
1594
1595 .opPutByValNotContiguous:
1596     bineq t2, ArrayStorageShape, .opPutByValSlow
1597     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1598     btqz ArrayStorage::m_vector[t0, t3, 8], .opPutByValArrayStorageEmpty
1599 .opPutByValArrayStorageStoreResult:
1600     loadisFromInstruction(3, t2)
1601     loadConstantOrVariable(t2, t1)
1602     storeq t1, ArrayStorage::m_vector[t0, t3, 8]
1603     dispatch(5)
1604
1605 .opPutByValArrayStorageEmpty:
1606     loadpFromInstruction(4, t1)
1607     storeb 1, ArrayProfile::m_mayStoreToHole[t1]
1608     addi 1, ArrayStorage::m_numValuesInVector[t0]
1609     bib t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .opPutByValArrayStorageStoreResult
1610     addi 1, t3, t1
1611     storei t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1612     jmp .opPutByValArrayStorageStoreResult
1613
1614 .opPutByValOutOfBounds:
1615     loadpFromInstruction(4, t0)
1616     storeb 1, ArrayProfile::m_outOfBounds[t0]
1617 .opPutByValSlow:
1618     callSlowPath(slowPath)
1619     dispatch(5)
1620 end
1621
1622 _llint_op_put_by_val:
1623     putByVal(_llint_slow_path_put_by_val)
1624
1625 _llint_op_put_by_val_direct:
1626     putByVal(_llint_slow_path_put_by_val_direct)
1627
1628
1629 _llint_op_jmp:
1630     traceExecution()
1631     dispatchIntIndirect(1)
1632
1633
1634 macro jumpTrueOrFalse(conditionOp, slow)
1635     loadisFromInstruction(1, t1)
1636     loadConstantOrVariable(t1, t0)
1637     xorq ValueFalse, t0
1638     btqnz t0, -1, .slow
1639     conditionOp(t0, .target)
1640     dispatch(3)
1641
1642 .target:
1643     dispatchIntIndirect(2)
1644
1645 .slow:
1646     callSlowPath(slow)
1647     dispatch(0)
1648 end
1649
1650
1651 macro equalNull(cellHandler, immediateHandler)
1652     loadisFromInstruction(1, t0)
1653     assertNotConstant(t0)
1654     loadq [cfr, t0, 8], t0
1655     btqnz t0, tagMask, .immediate
1656     loadStructureWithScratch(t0, t2, t1)
1657     cellHandler(t2, JSCell::m_flags[t0], .target)
1658     dispatch(3)
1659
1660 .target:
1661     dispatchIntIndirect(2)
1662
1663 .immediate:
1664     andq ~TagBitUndefined, t0
1665     immediateHandler(t0, .target)
1666     dispatch(3)
1667 end
1668
1669 _llint_op_jeq_null:
1670     traceExecution()
1671     equalNull(
1672         macro (structure, value, target) 
1673             btbz value, MasqueradesAsUndefined, .notMasqueradesAsUndefined
1674             loadp CodeBlock[cfr], t0
1675             loadp CodeBlock::m_globalObject[t0], t0
1676             bpeq Structure::m_globalObject[structure], t0, target
1677 .notMasqueradesAsUndefined:
1678         end,
1679         macro (value, target) bqeq value, ValueNull, target end)
1680
1681
1682 _llint_op_jneq_null:
1683     traceExecution()
1684     equalNull(
1685         macro (structure, value, target) 
1686             btbz value, MasqueradesAsUndefined, target
1687             loadp CodeBlock[cfr], t0
1688             loadp CodeBlock::m_globalObject[t0], t0
1689             bpneq Structure::m_globalObject[structure], t0, target
1690         end,
1691         macro (value, target) bqneq value, ValueNull, target end)
1692
1693
1694 _llint_op_jneq_ptr:
1695     traceExecution()
1696     loadisFromInstruction(1, t0)
1697     loadisFromInstruction(2, t1)
1698     loadp CodeBlock[cfr], t2
1699     loadp CodeBlock::m_globalObject[t2], t2
1700     loadp JSGlobalObject::m_specialPointers[t2, t1, 8], t1
1701     bpneq t1, [cfr, t0, 8], .opJneqPtrTarget
1702     dispatch(4)
1703
1704 .opJneqPtrTarget:
1705     dispatchIntIndirect(3)
1706
1707
1708 macro compare(integerCompare, doubleCompare, slowPath)
1709     loadisFromInstruction(1, t2)
1710     loadisFromInstruction(2, t3)
1711     loadConstantOrVariable(t2, t0)
1712     loadConstantOrVariable(t3, t1)
1713     bqb t0, tagTypeNumber, .op1NotInt
1714     bqb t1, tagTypeNumber, .op2NotInt
1715     integerCompare(t0, t1, .jumpTarget)
1716     dispatch(4)
1717
1718 .op1NotInt:
1719     btqz t0, tagTypeNumber, .slow
1720     bqb t1, tagTypeNumber, .op1NotIntOp2NotInt
1721     ci2d t1, ft1
1722     jmp .op1NotIntReady
1723 .op1NotIntOp2NotInt:
1724     btqz t1, tagTypeNumber, .slow
1725     addq tagTypeNumber, t1
1726     fq2d t1, ft1
1727 .op1NotIntReady:
1728     addq tagTypeNumber, t0
1729     fq2d t0, ft0
1730     doubleCompare(ft0, ft1, .jumpTarget)
1731     dispatch(4)
1732
1733 .op2NotInt:
1734     ci2d t0, ft0
1735     btqz t1, tagTypeNumber, .slow
1736     addq tagTypeNumber, t1
1737     fq2d t1, ft1
1738     doubleCompare(ft0, ft1, .jumpTarget)
1739     dispatch(4)
1740
1741 .jumpTarget:
1742     dispatchIntIndirect(3)
1743
1744 .slow:
1745     callSlowPath(slowPath)
1746     dispatch(0)
1747 end
1748
1749
1750 _llint_op_switch_imm:
1751     traceExecution()
1752     loadisFromInstruction(3, t2)
1753     loadisFromInstruction(1, t3)
1754     loadConstantOrVariable(t2, t1)
1755     loadp CodeBlock[cfr], t2
1756     loadp CodeBlock::m_rareData[t2], t2
1757     muli sizeof SimpleJumpTable, t3    # FIXME: would be nice to peephole this!
1758     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1759     addp t3, t2
1760     bqb t1, tagTypeNumber, .opSwitchImmNotInt
1761     subi SimpleJumpTable::min[t2], t1
1762     biaeq t1, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchImmFallThrough
1763     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t3
1764     loadis [t3, t1, 4], t1
1765     btiz t1, .opSwitchImmFallThrough
1766     dispatch(t1)
1767
1768 .opSwitchImmNotInt:
1769     btqnz t1, tagTypeNumber, .opSwitchImmSlow   # Go slow if it's a double.
1770 .opSwitchImmFallThrough:
1771     dispatchIntIndirect(2)
1772
1773 .opSwitchImmSlow:
1774     callSlowPath(_llint_slow_path_switch_imm)
1775     dispatch(0)
1776
1777
1778 _llint_op_switch_char:
1779     traceExecution()
1780     loadisFromInstruction(3, t2)
1781     loadisFromInstruction(1, t3)
1782     loadConstantOrVariable(t2, t1)
1783     loadp CodeBlock[cfr], t2
1784     loadp CodeBlock::m_rareData[t2], t2
1785     muli sizeof SimpleJumpTable, t3
1786     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1787     addp t3, t2
1788     btqnz t1, tagMask, .opSwitchCharFallThrough
1789     bbneq JSCell::m_type[t1], StringType, .opSwitchCharFallThrough
1790     bineq JSString::m_length[t1], 1, .opSwitchCharFallThrough
1791     loadp JSString::m_value[t1], t0
1792     btpz  t0, .opSwitchOnRope
1793     loadp StringImpl::m_data8[t0], t1
1794     btinz StringImpl::m_hashAndFlags[t0], HashFlags8BitBuffer, .opSwitchChar8Bit
1795     loadh [t1], t0
1796     jmp .opSwitchCharReady
1797 .opSwitchChar8Bit:
1798     loadb [t1], t0
1799 .opSwitchCharReady:
1800     subi SimpleJumpTable::min[t2], t0
1801     biaeq t0, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchCharFallThrough
1802     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t2
1803     loadis [t2, t0, 4], t1
1804     btiz t1, .opSwitchCharFallThrough
1805     dispatch(t1)
1806
1807 .opSwitchCharFallThrough:
1808     dispatchIntIndirect(2)
1809
1810 .opSwitchOnRope:
1811     callSlowPath(_llint_slow_path_switch_char)
1812     dispatch(0)
1813
1814
1815 macro arrayProfileForCall()
1816     loadisFromInstruction(4, t3)
1817     negp t3
1818     loadq ThisArgumentOffset[cfr, t3, 8], t0
1819     btqnz t0, tagMask, .done
1820     loadpFromInstruction((CallOpCodeSize - 2), t1)
1821     loadi JSCell::m_structureID[t0], t3
1822     storei t3, ArrayProfile::m_lastSeenStructureID[t1]
1823 .done:
1824 end
1825
1826 macro doCall(slowPath, prepareCall)
1827     loadisFromInstruction(2, t0)
1828     loadpFromInstruction(5, t1)
1829     loadp LLIntCallLinkInfo::callee[t1], t2
1830     loadConstantOrVariable(t0, t3)
1831     bqneq t3, t2, .opCallSlow
1832     loadisFromInstruction(4, t3)
1833     lshifti 3, t3
1834     negp t3
1835     addp cfr, t3
1836     storeq t2, Callee[t3]
1837     loadisFromInstruction(3, t2)
1838     storei PC, ArgumentCount + TagOffset[cfr]
1839     storei t2, ArgumentCount + PayloadOffset[t3]
1840     move t3, sp
1841     prepareCall(LLIntCallLinkInfo::machineCodeTarget[t1], t2, t3, t4)
1842     callTargetFunction(LLIntCallLinkInfo::machineCodeTarget[t1])
1843
1844 .opCallSlow:
1845     slowPathForCall(slowPath, prepareCall)
1846 end
1847
1848 _llint_op_ret:
1849     traceExecution()
1850     checkSwitchToJITForEpilogue()
1851     loadisFromInstruction(1, t2)
1852     loadConstantOrVariable(t2, r0)
1853     doReturn()
1854
1855
1856 _llint_op_to_primitive:
1857     traceExecution()
1858     loadisFromInstruction(2, t2)
1859     loadisFromInstruction(1, t3)
1860     loadConstantOrVariable(t2, t0)
1861     btqnz t0, tagMask, .opToPrimitiveIsImm
1862     bbaeq JSCell::m_type[t0], ObjectType, .opToPrimitiveSlowCase
1863 .opToPrimitiveIsImm:
1864     storeq t0, [cfr, t3, 8]
1865     dispatch(3)
1866
1867 .opToPrimitiveSlowCase:
1868     callSlowPath(_slow_path_to_primitive)
1869     dispatch(3)
1870
1871
1872 _llint_op_catch:
1873     # This is where we end up from the JIT's throw trampoline (because the
1874     # machine code return address will be set to _llint_op_catch), and from
1875     # the interpreter's throw trampoline (see _llint_throw_trampoline).
1876     # The throwing code must have known that we were throwing to the interpreter,
1877     # and have set VM::targetInterpreterPCForThrow.
1878     loadp Callee[cfr], t3
1879     andp MarkedBlockMask, t3
1880     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1881     restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(t3, t0)
1882     loadp VM::callFrameForCatch[t3], cfr
1883     storep 0, VM::callFrameForCatch[t3]
1884     restoreStackPointerAfterCall()
1885
1886     loadp CodeBlock[cfr], PB
1887     loadp CodeBlock::m_instructions[PB], PB
1888     loadp VM::targetInterpreterPCForThrow[t3], PC
1889     subp PB, PC
1890     rshiftp 3, PC
1891
1892     callSlowPath(_llint_slow_path_check_if_exception_is_uncatchable_and_notify_profiler)
1893     bpeq r1, 0, .isCatchableException
1894     jmp _llint_throw_from_slow_path_trampoline
1895
1896 .isCatchableException:
1897     loadp Callee[cfr], t3
1898     andp MarkedBlockMask, t3
1899     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1900
1901     loadq VM::m_exception[t3], t0
1902     storeq 0, VM::m_exception[t3]
1903     loadisFromInstruction(1, t2)
1904     storeq t0, [cfr, t2, 8]
1905
1906     loadq Exception::m_value[t0], t3
1907     loadisFromInstruction(2, t2)
1908     storeq t3, [cfr, t2, 8]
1909
1910     traceExecution()
1911     dispatch(3)
1912
1913
1914 _llint_op_end:
1915     traceExecution()
1916     checkSwitchToJITForEpilogue()
1917     loadisFromInstruction(1, t0)
1918     assertNotConstant(t0)
1919     loadq [cfr, t0, 8], r0
1920     doReturn()
1921
1922
1923 _llint_throw_from_slow_path_trampoline:
1924     loadp Callee[cfr], t1
1925     andp MarkedBlockMask, t1
1926     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1927     copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(t1, t2)
1928
1929     callSlowPath(_llint_slow_path_handle_exception)
1930
1931     # When throwing from the interpreter (i.e. throwing from LLIntSlowPaths), so
1932     # the throw target is not necessarily interpreted code, we come to here.
1933     # This essentially emulates the JIT's throwing protocol.
1934     loadp Callee[cfr], t1
1935     andp MarkedBlockMask, t1
1936     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1937     jmp VM::targetMachinePCForThrow[t1]
1938
1939
1940 _llint_throw_during_call_trampoline:
1941     preserveReturnAddressAfterCall(t2)
1942     jmp _llint_throw_from_slow_path_trampoline
1943
1944
1945 macro nativeCallTrampoline(executableOffsetToFunction)
1946
1947     functionPrologue()
1948     storep 0, CodeBlock[cfr]
1949     loadp Callee[cfr], t0
1950     andp MarkedBlockMask, t0, t1
1951     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1952     storep cfr, VM::topCallFrame[t1]
1953     if ARM64 or C_LOOP
1954         storep lr, ReturnPC[cfr]
1955     end
1956     move cfr, a0
1957     loadp Callee[cfr], t1
1958     loadp JSFunction::m_executable[t1], t1
1959     checkStackPointerAlignment(t3, 0xdead0001)
1960     if C_LOOP
1961         cloopCallNative executableOffsetToFunction[t1]
1962     else
1963         if X86_64_WIN
1964             subp 32, sp
1965         end
1966         call executableOffsetToFunction[t1]
1967         if X86_64_WIN
1968             addp 32, sp
1969         end
1970     end
1971     loadp Callee[cfr], t3
1972     andp MarkedBlockMask, t3
1973     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1974
1975     functionEpilogue()
1976
1977     btqnz VM::m_exception[t3], .handleException
1978     ret
1979
1980 .handleException:
1981     storep cfr, VM::topCallFrame[t3]
1982     restoreStackPointerAfterCall()
1983     jmp _llint_throw_from_slow_path_trampoline
1984 end
1985
1986 macro getConstantScope(dst)
1987     loadpFromInstruction(6, t0)
1988     loadisFromInstruction(dst, t1)
1989     storeq t0, [cfr, t1, 8]
1990 end
1991
1992 macro varInjectionCheck(slowPath)
1993     loadp CodeBlock[cfr], t0
1994     loadp CodeBlock::m_globalObject[t0], t0
1995     loadp JSGlobalObject::m_varInjectionWatchpoint[t0], t0
1996     bbeq WatchpointSet::m_state[t0], IsInvalidated, slowPath
1997 end
1998
1999 macro resolveScope()
2000     loadisFromInstruction(5, t2)
2001     loadisFromInstruction(2, t0)
2002     loadp [cfr, t0, 8], t0
2003     btiz t2, .resolveScopeLoopEnd
2004
2005 .resolveScopeLoop:
2006     loadp JSScope::m_next[t0], t0
2007     subi 1, t2
2008     btinz t2, .resolveScopeLoop
2009
2010 .resolveScopeLoopEnd:
2011     loadisFromInstruction(1, t1)
2012     storeq t0, [cfr, t1, 8]
2013 end
2014
2015
2016 _llint_op_resolve_scope:
2017     traceExecution()
2018     loadisFromInstruction(4, t0)
2019
2020 #rGlobalProperty:
2021     bineq t0, GlobalProperty, .rGlobalVar
2022     getConstantScope(1)
2023     dispatch(7)
2024
2025 .rGlobalVar:
2026     bineq t0, GlobalVar, .rGlobalLexicalVar
2027     getConstantScope(1)
2028     dispatch(7)
2029
2030 .rGlobalLexicalVar:
2031     bineq t0, GlobalLexicalVar, .rClosureVar
2032     getConstantScope(1)
2033     dispatch(7)
2034
2035 .rClosureVar:
2036     bineq t0, ClosureVar, .rModuleVar
2037     resolveScope()
2038     dispatch(7)
2039
2040 .rModuleVar:
2041     bineq t0, ModuleVar, .rGlobalPropertyWithVarInjectionChecks
2042     getConstantScope(1)
2043     dispatch(7)
2044
2045 .rGlobalPropertyWithVarInjectionChecks:
2046     bineq t0, GlobalPropertyWithVarInjectionChecks, .rGlobalVarWithVarInjectionChecks
2047     varInjectionCheck(.rDynamic)
2048     getConstantScope(1)
2049     dispatch(7)
2050
2051 .rGlobalVarWithVarInjectionChecks:
2052     bineq t0, GlobalVarWithVarInjectionChecks, .rGlobalLexicalVarWithVarInjectionChecks
2053     varInjectionCheck(.rDynamic)
2054     getConstantScope(1)
2055     dispatch(7)
2056
2057 .rGlobalLexicalVarWithVarInjectionChecks:
2058     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .rClosureVarWithVarInjectionChecks
2059     varInjectionCheck(.rDynamic)
2060     getConstantScope(1)
2061     dispatch(7)
2062
2063 .rClosureVarWithVarInjectionChecks:
2064     bineq t0, ClosureVarWithVarInjectionChecks, .rDynamic
2065     varInjectionCheck(.rDynamic)
2066     resolveScope()
2067     dispatch(7)
2068
2069 .rDynamic:
2070     callSlowPath(_slow_path_resolve_scope)
2071     dispatch(7)
2072
2073
2074 macro loadWithStructureCheck(operand, slowPath)
2075     loadisFromInstruction(operand, t0)
2076     loadq [cfr, t0, 8], t0
2077     loadStructureWithScratch(t0, t2, t1)
2078     loadpFromInstruction(5, t1)
2079     bpneq t2, t1, slowPath
2080 end
2081
2082 macro getProperty()
2083     loadisFromInstruction(6, t1)
2084     loadPropertyAtVariableOffset(t1, t0, t2)
2085     valueProfile(t2, 7, t0)
2086     loadisFromInstruction(1, t0)
2087     storeq t2, [cfr, t0, 8]
2088 end
2089
2090 macro getGlobalVar(tdzCheckIfNecessary)
2091     loadpFromInstruction(6, t0)
2092     loadq [t0], t0
2093     tdzCheckIfNecessary(t0)
2094     valueProfile(t0, 7, t1)
2095     loadisFromInstruction(1, t1)
2096     storeq t0, [cfr, t1, 8]
2097 end
2098
2099 macro getClosureVar()
2100     loadisFromInstruction(6, t1)
2101     loadq JSEnvironmentRecord_variables[t0, t1, 8], t0
2102     valueProfile(t0, 7, t1)
2103     loadisFromInstruction(1, t1)
2104     storeq t0, [cfr, t1, 8]
2105 end
2106
2107 _llint_op_get_from_scope:
2108     traceExecution()
2109     loadisFromInstruction(4, t0)
2110     andi ResolveTypeMask, t0
2111
2112 #gGlobalProperty:
2113     bineq t0, GlobalProperty, .gGlobalVar
2114     loadWithStructureCheck(2, .gDynamic)
2115     getProperty()
2116     dispatch(8)
2117
2118 .gGlobalVar:
2119     bineq t0, GlobalVar, .gGlobalLexicalVar
2120     getGlobalVar(macro(v) end)
2121     dispatch(8)
2122
2123 .gGlobalLexicalVar:
2124     bineq t0, GlobalLexicalVar, .gClosureVar
2125     getGlobalVar(
2126         macro (value)
2127             bqeq value, ValueEmpty, .gDynamic
2128         end)
2129     dispatch(8)
2130
2131 .gClosureVar:
2132     bineq t0, ClosureVar, .gGlobalPropertyWithVarInjectionChecks
2133     loadVariable(2, t0)
2134     getClosureVar()
2135     dispatch(8)
2136
2137 .gGlobalPropertyWithVarInjectionChecks:
2138     bineq t0, GlobalPropertyWithVarInjectionChecks, .gGlobalVarWithVarInjectionChecks
2139     loadWithStructureCheck(2, .gDynamic)
2140     getProperty()
2141     dispatch(8)
2142
2143 .gGlobalVarWithVarInjectionChecks:
2144     bineq t0, GlobalVarWithVarInjectionChecks, .gGlobalLexicalVarWithVarInjectionChecks
2145     varInjectionCheck(.gDynamic)
2146     getGlobalVar(macro(v) end)
2147     dispatch(8)
2148
2149 .gGlobalLexicalVarWithVarInjectionChecks:
2150     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .gClosureVarWithVarInjectionChecks
2151     varInjectionCheck(.gDynamic)
2152     getGlobalVar(
2153         macro (value)
2154             bqeq value, ValueEmpty, .gDynamic
2155         end)
2156     dispatch(8)
2157
2158 .gClosureVarWithVarInjectionChecks:
2159     bineq t0, ClosureVarWithVarInjectionChecks, .gDynamic
2160     varInjectionCheck(.gDynamic)
2161     loadVariable(2, t0)
2162     getClosureVar()
2163     dispatch(8)
2164
2165 .gDynamic:
2166     callSlowPath(_llint_slow_path_get_from_scope)
2167     dispatch(8)
2168
2169
2170 macro putProperty()
2171     loadisFromInstruction(3, t1)
2172     loadConstantOrVariable(t1, t2)
2173     loadisFromInstruction(6, t1)
2174     storePropertyAtVariableOffset(t1, t0, t2)
2175 end
2176
2177 macro putGlobalVariable()
2178     loadisFromInstruction(3, t0)
2179     loadConstantOrVariable(t0, t1)
2180     loadpFromInstruction(5, t2)
2181     loadpFromInstruction(6, t0)
2182     notifyWrite(t2, .pDynamic)
2183     storeq t1, [t0]
2184 end
2185
2186 macro putClosureVar()
2187     loadisFromInstruction(3, t1)
2188     loadConstantOrVariable(t1, t2)
2189     loadisFromInstruction(6, t1)
2190     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2191 end
2192
2193 macro putLocalClosureVar()
2194     loadisFromInstruction(3, t1)
2195     loadConstantOrVariable(t1, t2)
2196     loadpFromInstruction(5, t3)
2197     btpz t3, .noVariableWatchpointSet
2198     notifyWrite(t3, .pDynamic)
2199 .noVariableWatchpointSet:
2200     loadisFromInstruction(6, t1)
2201     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2202 end
2203
2204 macro checkTDZInGlobalPutToScopeIfNecessary()
2205     loadisFromInstruction(4, t0)
2206     andi InitializationModeMask, t0
2207     rshifti InitializationModeShift, t0
2208     bineq t0, NotInitialization, .noNeedForTDZCheck
2209     loadpFromInstruction(6, t0)
2210     loadq [t0], t0
2211     bqeq t0, ValueEmpty, .pDynamic
2212 .noNeedForTDZCheck:
2213 end
2214
2215
2216 _llint_op_put_to_scope:
2217     traceExecution()
2218     loadisFromInstruction(4, t0)
2219     andi ResolveTypeMask, t0
2220
2221 #pLocalClosureVar:
2222     bineq t0, LocalClosureVar, .pGlobalProperty
2223     writeBarrierOnOperands(1, 3)
2224     loadVariable(1, t0)
2225     putLocalClosureVar()
2226     dispatch(7)
2227
2228 .pGlobalProperty:
2229     bineq t0, GlobalProperty, .pGlobalVar
2230     writeBarrierOnOperands(1, 3)
2231     loadWithStructureCheck(1, .pDynamic)
2232     putProperty()
2233     dispatch(7)
2234
2235 .pGlobalVar:
2236     bineq t0, GlobalVar, .pGlobalLexicalVar
2237     writeBarrierOnGlobalObject(3)
2238     putGlobalVariable()
2239     dispatch(7)
2240
2241 .pGlobalLexicalVar:
2242     bineq t0, GlobalLexicalVar, .pClosureVar
2243     writeBarrierOnGlobalLexicalEnvironment(3)
2244     checkTDZInGlobalPutToScopeIfNecessary()
2245     putGlobalVariable()
2246     dispatch(7)
2247
2248 .pClosureVar:
2249     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
2250     writeBarrierOnOperands(1, 3)
2251     loadVariable(1, t0)
2252     putClosureVar()
2253     dispatch(7)
2254
2255 .pGlobalPropertyWithVarInjectionChecks:
2256     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
2257     writeBarrierOnOperands(1, 3)
2258     loadWithStructureCheck(1, .pDynamic)
2259     putProperty()
2260     dispatch(7)
2261
2262 .pGlobalVarWithVarInjectionChecks:
2263     bineq t0, GlobalVarWithVarInjectionChecks, .pGlobalLexicalVarWithVarInjectionChecks
2264     writeBarrierOnGlobalObject(3)
2265     varInjectionCheck(.pDynamic)
2266     putGlobalVariable()
2267     dispatch(7)
2268
2269 .pGlobalLexicalVarWithVarInjectionChecks:
2270     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
2271     writeBarrierOnGlobalLexicalEnvironment(3)
2272     varInjectionCheck(.pDynamic)
2273     checkTDZInGlobalPutToScopeIfNecessary()
2274     putGlobalVariable()
2275     dispatch(7)
2276
2277 .pClosureVarWithVarInjectionChecks:
2278     bineq t0, ClosureVarWithVarInjectionChecks, .pModuleVar
2279     writeBarrierOnOperands(1, 3)
2280     varInjectionCheck(.pDynamic)
2281     loadVariable(1, t0)
2282     putClosureVar()
2283     dispatch(7)
2284
2285 .pModuleVar:
2286     bineq t0, ModuleVar, .pDynamic
2287     callSlowPath(_slow_path_throw_strict_mode_readonly_property_write_error)
2288     dispatch(7)
2289
2290 .pDynamic:
2291     callSlowPath(_llint_slow_path_put_to_scope)
2292     dispatch(7)
2293
2294
2295 _llint_op_get_from_arguments:
2296     traceExecution()
2297     loadVariable(2, t0)
2298     loadi 24[PB, PC, 8], t1
2299     loadq DirectArguments_storage[t0, t1, 8], t0
2300     valueProfile(t0, 4, t1)
2301     loadisFromInstruction(1, t1)
2302     storeq t0, [cfr, t1, 8]
2303     dispatch(5)
2304
2305
2306 _llint_op_put_to_arguments:
2307     traceExecution()
2308     writeBarrierOnOperands(1, 3)
2309     loadVariable(1, t0)
2310     loadi 16[PB, PC, 8], t1
2311     loadisFromInstruction(3, t3)
2312     loadConstantOrVariable(t3, t2)
2313     storeq t2, DirectArguments_storage[t0, t1, 8]
2314     dispatch(4)
2315
2316
2317 _llint_op_get_parent_scope:
2318     traceExecution()
2319     loadVariable(2, t0)
2320     loadp JSScope::m_next[t0], t0
2321     loadisFromInstruction(1, t1)
2322     storeq t0, [cfr, t1, 8]
2323     dispatch(3)
2324
2325
2326 _llint_op_profile_type:
2327     traceExecution()
2328     loadp CodeBlock[cfr], t1
2329     loadp CodeBlock::m_vm[t1], t1
2330     # t1 is holding the pointer to the typeProfilerLog.
2331     loadp VM::m_typeProfilerLog[t1], t1
2332     # t2 is holding the pointer to the current log entry.
2333     loadp TypeProfilerLog::m_currentLogEntryPtr[t1], t2
2334
2335     # t0 is holding the JSValue argument.
2336     loadisFromInstruction(1, t3)
2337     loadConstantOrVariable(t3, t0)
2338
2339     bqeq t0, ValueEmpty, .opProfileTypeDone
2340     # Store the JSValue onto the log entry.
2341     storeq t0, TypeProfilerLog::LogEntry::value[t2]
2342     
2343     # Store the TypeLocation onto the log entry.
2344     loadpFromInstruction(2, t3)
2345     storep t3, TypeProfilerLog::LogEntry::location[t2]
2346
2347     btqz t0, tagMask, .opProfileTypeIsCell
2348     storei 0, TypeProfilerLog::LogEntry::structureID[t2]
2349     jmp .opProfileTypeSkipIsCell
2350 .opProfileTypeIsCell:
2351     loadi JSCell::m_structureID[t0], t3
2352     storei t3, TypeProfilerLog::LogEntry::structureID[t2]
2353 .opProfileTypeSkipIsCell:
2354     
2355     # Increment the current log entry.
2356     addp sizeof TypeProfilerLog::LogEntry, t2
2357     storep t2, TypeProfilerLog::m_currentLogEntryPtr[t1]
2358
2359     loadp TypeProfilerLog::m_logEndPtr[t1], t1
2360     bpneq t2, t1, .opProfileTypeDone
2361     callSlowPath(_slow_path_profile_type_clear_log)
2362
2363 .opProfileTypeDone:
2364     dispatch(6)
2365
2366 _llint_op_profile_control_flow:
2367     traceExecution()
2368     loadpFromInstruction(1, t0)
2369     addq 1, BasicBlockLocation::m_executionCount[t0]
2370     dispatch(2)
2371
2372
2373 _llint_op_get_rest_length:
2374     traceExecution()
2375     loadi PayloadOffset + ArgumentCount[cfr], t0
2376     subi 1, t0
2377     loadisFromInstruction(2, t1)
2378     bilteq t0, t1, .storeZero
2379     subi t1, t0
2380     jmp .boxUp
2381 .storeZero:
2382     move 0, t0
2383 .boxUp:
2384     orq tagTypeNumber, t0
2385     loadisFromInstruction(1, t1)
2386     storeq t0, [cfr, t1, 8]
2387     dispatch(3)
2388
2389
2390 _llint_op_log_shadow_chicken_prologue:
2391     traceExecution()
2392     acquireShadowChickenPacket(.opLogShadowChickenPrologueSlow)
2393     storep cfr, ShadowChicken::Packet::frame[t0]
2394     loadp CallerFrame[cfr], t1
2395     storep t1, ShadowChicken::Packet::callerFrame[t0]
2396     loadp Callee + PayloadOffset[cfr], t1
2397     storep t1, ShadowChicken::Packet::callee[t0]
2398     loadVariable(1, t1)
2399     storep t1, ShadowChicken::Packet::scope[t0]
2400     dispatch(2)
2401 .opLogShadowChickenPrologueSlow:
2402     callSlowPath(_llint_slow_path_log_shadow_chicken_prologue)
2403     dispatch(2)
2404
2405
2406 _llint_op_log_shadow_chicken_tail:
2407     traceExecution()
2408     acquireShadowChickenPacket(.opLogShadowChickenTailSlow)
2409     storep cfr, ShadowChicken::Packet::frame[t0]
2410     storep ShadowChickenTailMarker, ShadowChicken::Packet::callee[t0]
2411     loadVariable(1, t1)
2412     storep t1, ShadowChicken::Packet::thisValue[t0]
2413     loadVariable(2, t1)
2414     storep t1, ShadowChicken::Packet::scope[t0]
2415     loadp CodeBlock[cfr], t1
2416     storep t1, ShadowChicken::Packet::codeBlock[t0]
2417     storei PC, ShadowChicken::Packet::callSiteIndex[t0]
2418     dispatch(3)
2419 .opLogShadowChickenTailSlow:
2420     callSlowPath(_llint_slow_path_log_shadow_chicken_tail)
2421     dispatch(3)