2763303a2da3fd1c7147b1448608ecd8a39c759e
[WebKit-https.git] / Source / JavaScriptCore / llint / LowLevelInterpreter64.asm
1 # Copyright (C) 2011-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1. Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2. Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
13 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
14 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
16 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
17 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
18 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
19 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
20 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
22 # THE POSSIBILITY OF SUCH DAMAGE.
23
24
25 # Utilities.
26 macro jumpToInstruction()
27     jmp [PB, PC, 8]
28 end
29
30 macro dispatch(advance)
31     addp advance, PC
32     jumpToInstruction()
33 end
34
35 macro dispatchInt(advance)
36     addi advance, PC
37     jumpToInstruction()
38 end
39
40 macro dispatchIntIndirect(offset)
41     dispatchInt(offset * 8[PB, PC, 8])
42 end
43
44 macro dispatchAfterCall()
45     loadi ArgumentCount + TagOffset[cfr], PC
46     loadp CodeBlock[cfr], PB
47     loadp CodeBlock::m_instructions[PB], PB
48     loadisFromInstruction(1, t1)
49     storeq r0, [cfr, t1, 8]
50     valueProfile(r0, (CallOpCodeSize - 1), t3)
51     dispatch(CallOpCodeSize)
52 end
53
54 macro cCall2(function)
55     checkStackPointerAlignment(t4, 0xbad0c002)
56     if X86_64 or ARM64
57         call function
58     elsif X86_64_WIN
59         # Note: this implementation is only correct if the return type size is > 8 bytes.
60         # See macro cCall2Void for an implementation when the return type <= 8 bytes.
61         # On Win64, when the return type is larger than 8 bytes, we need to allocate space on the stack for the return value.
62         # On entry rcx (a0), should contain a pointer to this stack space. The other parameters are shifted to the right,
63         # rdx (a1) should contain the first argument, and r8 (a2) should contain the second argument.
64         # On return, rax contains a pointer to this stack value, and we then need to copy the 16 byte return value into rax (r0) and rdx (r1)
65         # since the return value is expected to be split between the two.
66         # See http://msdn.microsoft.com/en-us/library/7572ztz4.aspx
67         move a1, a2
68         move a0, a1
69         subp 48, sp
70         move sp, a0
71         addp 32, a0
72         call function
73         addp 48, sp
74         move 8[r0], r1
75         move [r0], r0
76     elsif C_LOOP
77         cloopCallSlowPath function, a0, a1
78     else
79         error
80     end
81 end
82
83 macro cCall2Void(function)
84     if C_LOOP
85         cloopCallSlowPathVoid function, a0, a1
86     elsif X86_64_WIN
87         # Note: we cannot use the cCall2 macro for Win64 in this case,
88         # as the Win64 cCall2 implemenation is only correct when the return type size is > 8 bytes.
89         # On Win64, rcx and rdx are used for passing the first two parameters.
90         # We also need to make room on the stack for all four parameter registers.
91         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
92         subp 32, sp 
93         call function
94         addp 32, sp 
95     else
96         cCall2(function)
97     end
98 end
99
100 # This barely works. arg3 and arg4 should probably be immediates.
101 macro cCall4(function)
102     checkStackPointerAlignment(t4, 0xbad0c004)
103     if X86_64 or ARM64
104         call function
105     elsif X86_64_WIN
106         # On Win64, rcx, rdx, r8, and r9 are used for passing the first four parameters.
107         # We also need to make room on the stack for all four parameter registers.
108         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
109         subp 64, sp
110         call function
111         addp 64, sp
112     else
113         error
114     end
115 end
116
117 macro doVMEntry(makeCall)
118     functionPrologue()
119     pushCalleeSaves()
120
121     const entry = a0
122     const vm = a1
123     const protoCallFrame = a2
124
125     vmEntryRecord(cfr, sp)
126
127     checkStackPointerAlignment(t4, 0xbad0dc01)
128
129     storep vm, VMEntryRecord::m_vm[sp]
130     loadp VM::topCallFrame[vm], t4
131     storep t4, VMEntryRecord::m_prevTopCallFrame[sp]
132     loadp VM::topVMEntryFrame[vm], t4
133     storep t4, VMEntryRecord::m_prevTopVMEntryFrame[sp]
134
135     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t4
136     addp CallFrameHeaderSlots, t4, t4
137     lshiftp 3, t4
138     subp sp, t4, t3
139
140     # Ensure that we have enough additional stack capacity for the incoming args,
141     # and the frame for the JS code we're executing. We need to do this check
142     # before we start copying the args from the protoCallFrame below.
143     if C_LOOP
144         bpaeq t3, VM::m_cloopStackLimit[vm], .stackHeightOK
145     else
146         bpaeq t3, VM::m_osStackLimitWithReserve[vm], .stackHeightOK
147     end
148
149     if C_LOOP
150         move entry, t4
151         move vm, t5
152         cloopCallSlowPath _llint_stack_check_at_vm_entry, vm, t3
153         bpeq t0, 0, .stackCheckFailed
154         move t4, entry
155         move t5, vm
156         jmp .stackHeightOK
157
158 .stackCheckFailed:
159         move t4, entry
160         move t5, vm
161     end
162
163     move vm, a0
164     move protoCallFrame, a1
165     cCall2(_llint_throw_stack_overflow_error)
166
167     vmEntryRecord(cfr, t4)
168
169     loadp VMEntryRecord::m_vm[t4], vm
170     loadp VMEntryRecord::m_prevTopCallFrame[t4], extraTempReg
171     storep extraTempReg, VM::topCallFrame[vm]
172     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], extraTempReg
173     storep extraTempReg, VM::topVMEntryFrame[vm]
174
175     subp cfr, CalleeRegisterSaveSize, sp
176
177     popCalleeSaves()
178     functionEpilogue()
179     ret
180
181 .stackHeightOK:
182     move t3, sp
183     move 4, t3
184
185 .copyHeaderLoop:
186     subi 1, t3
187     loadq [protoCallFrame, t3, 8], extraTempReg
188     storeq extraTempReg, CodeBlock[sp, t3, 8]
189     btinz t3, .copyHeaderLoop
190
191     loadi PayloadOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], t4
192     subi 1, t4
193     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], extraTempReg
194     subi 1, extraTempReg
195
196     bieq t4, extraTempReg, .copyArgs
197     move ValueUndefined, t3
198 .fillExtraArgsLoop:
199     subi 1, extraTempReg
200     storeq t3, ThisArgumentOffset + 8[sp, extraTempReg, 8]
201     bineq t4, extraTempReg, .fillExtraArgsLoop
202
203 .copyArgs:
204     loadp ProtoCallFrame::args[protoCallFrame], t3
205
206 .copyArgsLoop:
207     btiz t4, .copyArgsDone
208     subi 1, t4
209     loadq [t3, t4, 8], extraTempReg
210     storeq extraTempReg, ThisArgumentOffset + 8[sp, t4, 8]
211     jmp .copyArgsLoop
212
213 .copyArgsDone:
214     if ARM64
215         move sp, t4
216         storep t4, VM::topCallFrame[vm]
217     else
218         storep sp, VM::topCallFrame[vm]
219     end
220     storep cfr, VM::topVMEntryFrame[vm]
221
222     checkStackPointerAlignment(extraTempReg, 0xbad0dc02)
223
224     makeCall(entry, t3)
225
226     # We may have just made a call into a JS function, so we can't rely on sp
227     # for anything but the fact that our own locals (ie the VMEntryRecord) are
228     # not below it. It also still has to be aligned, though.
229     checkStackPointerAlignment(t2, 0xbad0dc03)
230
231     vmEntryRecord(cfr, t4)
232
233     loadp VMEntryRecord::m_vm[t4], vm
234     loadp VMEntryRecord::m_prevTopCallFrame[t4], t2
235     storep t2, VM::topCallFrame[vm]
236     loadp VMEntryRecord::m_prevTopVMEntryFrame[t4], t2
237     storep t2, VM::topVMEntryFrame[vm]
238
239     subp cfr, CalleeRegisterSaveSize, sp
240
241     popCalleeSaves()
242     functionEpilogue()
243
244     ret
245 end
246
247
248 macro makeJavaScriptCall(entry, temp)
249     addp 16, sp
250     if C_LOOP
251         cloopCallJSFunction entry
252     else
253         call entry
254     end
255     subp 16, sp
256 end
257
258
259 macro makeHostFunctionCall(entry, temp)
260     move entry, temp
261     storep cfr, [sp]
262     move sp, a0
263     if C_LOOP
264         storep lr, 8[sp]
265         cloopCallNative temp
266     elsif X86_64_WIN
267         # We need to allocate 32 bytes on the stack for the shadow space.
268         subp 32, sp
269         call temp
270         addp 32, sp
271     else
272         call temp
273     end
274 end
275
276
277 _handleUncaughtException:
278     loadp Callee[cfr], t3
279     andp MarkedBlockMask, t3
280     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
281     restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(t3, t0)
282     loadp VM::callFrameForCatch[t3], cfr
283     storep 0, VM::callFrameForCatch[t3]
284
285     loadp CallerFrame[cfr], cfr
286     vmEntryRecord(cfr, t2)
287
288     loadp VMEntryRecord::m_vm[t2], t3
289     loadp VMEntryRecord::m_prevTopCallFrame[t2], extraTempReg
290     storep extraTempReg, VM::topCallFrame[t3]
291     loadp VMEntryRecord::m_prevTopVMEntryFrame[t2], extraTempReg
292     storep extraTempReg, VM::topVMEntryFrame[t3]
293
294     subp cfr, CalleeRegisterSaveSize, sp
295
296     popCalleeSaves()
297     functionEpilogue()
298     ret
299
300
301 macro prepareStateForCCall()
302     leap [PB, PC, 8], PC
303 end
304
305 macro restoreStateAfterCCall()
306     move r0, PC
307     subp PB, PC
308     rshiftp 3, PC
309 end
310
311 macro callSlowPath(slowPath)
312     prepareStateForCCall()
313     move cfr, a0
314     move PC, a1
315     cCall2(slowPath)
316     restoreStateAfterCCall()
317 end
318
319 macro traceOperand(fromWhere, operand)
320     prepareStateForCCall()
321     move fromWhere, a2
322     move operand, a3
323     move cfr, a0
324     move PC, a1
325     cCall4(_llint_trace_operand)
326     restoreStateAfterCCall()
327 end
328
329 macro traceValue(fromWhere, operand)
330     prepareStateForCCall()
331     move fromWhere, a2
332     move operand, a3
333     move cfr, a0
334     move PC, a1
335     cCall4(_llint_trace_value)
336     restoreStateAfterCCall()
337 end
338
339 # Call a slow path for call call opcodes.
340 macro callCallSlowPath(slowPath, action)
341     storei PC, ArgumentCount + TagOffset[cfr]
342     prepareStateForCCall()
343     move cfr, a0
344     move PC, a1
345     cCall2(slowPath)
346     action(r0, r1)
347 end
348
349 macro callWatchdogTimerHandler(throwHandler)
350     storei PC, ArgumentCount + TagOffset[cfr]
351     prepareStateForCCall()
352     move cfr, a0
353     move PC, a1
354     cCall2(_llint_slow_path_handle_watchdog_timer)
355     btpnz r0, throwHandler
356     loadi ArgumentCount + TagOffset[cfr], PC
357 end
358
359 macro checkSwitchToJITForLoop()
360     checkSwitchToJIT(
361         1,
362         macro()
363             storei PC, ArgumentCount + TagOffset[cfr]
364             prepareStateForCCall()
365             move cfr, a0
366             move PC, a1
367             cCall2(_llint_loop_osr)
368             btpz r0, .recover
369             move r1, sp
370             jmp r0
371         .recover:
372             loadi ArgumentCount + TagOffset[cfr], PC
373         end)
374 end
375
376 macro loadVariable(operand, value)
377     loadisFromInstruction(operand, value)
378     loadq [cfr, value, 8], value
379 end
380
381 # Index and value must be different registers. Index may be clobbered.
382 macro loadConstantOrVariable(index, value)
383     bpgteq index, FirstConstantRegisterIndex, .constant
384     loadq [cfr, index, 8], value
385     jmp .done
386 .constant:
387     loadp CodeBlock[cfr], value
388     loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value
389     subp FirstConstantRegisterIndex, index
390     loadq [value, index, 8], value
391 .done:
392 end
393
394 macro loadConstantOrVariableInt32(index, value, slow)
395     loadConstantOrVariable(index, value)
396     bqb value, tagTypeNumber, slow
397 end
398
399 macro loadConstantOrVariableCell(index, value, slow)
400     loadConstantOrVariable(index, value)
401     btqnz value, tagMask, slow
402 end
403
404 macro writeBarrierOnOperand(cellOperand)
405     loadisFromInstruction(cellOperand, t1)
406     loadConstantOrVariableCell(t1, t2, .writeBarrierDone)
407     skipIfIsRememberedOrInEden(t2, t1, t3, 
408         macro(cellState)
409             btbnz cellState, .writeBarrierDone
410             push PB, PC
411             move t2, a1 # t2 can be a0 (not on 64 bits, but better safe than sorry)
412             move cfr, a0
413             cCall2Void(_llint_write_barrier_slow)
414             pop PC, PB
415         end
416     )
417 .writeBarrierDone:
418 end
419
420 macro writeBarrierOnOperands(cellOperand, valueOperand)
421     loadisFromInstruction(valueOperand, t1)
422     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
423     btpz t0, .writeBarrierDone
424
425     writeBarrierOnOperand(cellOperand)
426 .writeBarrierDone:
427 end
428
429 macro writeBarrierOnGlobal(valueOperand, loadHelper)
430     loadisFromInstruction(valueOperand, t1)
431     loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
432     btpz t0, .writeBarrierDone
433
434     loadHelper(t3)
435     skipIfIsRememberedOrInEden(t3, t1, t2,
436         macro(gcData)
437             btbnz gcData, .writeBarrierDone
438             push PB, PC
439             move cfr, a0
440             move t3, a1
441             cCall2Void(_llint_write_barrier_slow)
442             pop PC, PB
443         end
444     )
445 .writeBarrierDone:
446 end
447
448 macro writeBarrierOnGlobalObject(valueOperand)
449     writeBarrierOnGlobal(valueOperand,
450         macro(registerToStoreGlobal)
451             loadp CodeBlock[cfr], registerToStoreGlobal
452             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
453         end)
454 end
455
456 macro writeBarrierOnGlobalLexicalEnvironment(valueOperand)
457     writeBarrierOnGlobal(valueOperand,
458         macro(registerToStoreGlobal)
459             loadp CodeBlock[cfr], registerToStoreGlobal
460             loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal
461             loadp JSGlobalObject::m_globalLexicalEnvironment[registerToStoreGlobal], registerToStoreGlobal
462         end)
463 end
464
465 macro valueProfile(value, operand, scratch)
466     loadpFromInstruction(operand, scratch)
467     storeq value, ValueProfile::m_buckets[scratch]
468 end
469
470 macro structureIDToStructureWithScratch(structureIDThenStructure, scratch)
471     loadp CodeBlock[cfr], scratch
472     loadp CodeBlock::m_vm[scratch], scratch
473     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[scratch], scratch
474     loadp [scratch, structureIDThenStructure, 8], structureIDThenStructure
475 end
476
477 macro loadStructureWithScratch(cell, structure, scratch)
478     loadi JSCell::m_structureID[cell], structure
479     structureIDToStructureWithScratch(structure, scratch)
480 end
481
482 macro loadStructureAndClobberFirstArg(cell, structure)
483     loadi JSCell::m_structureID[cell], structure
484     loadp CodeBlock[cfr], cell
485     loadp CodeBlock::m_vm[cell], cell
486     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[cell], cell
487     loadp [cell, structure, 8], structure
488 end
489
490 macro storeStructureWithTypeInfo(cell, structure, scratch)
491     loadq Structure::m_blob + StructureIDBlob::u.doubleWord[structure], scratch
492     storeq scratch, JSCell::m_structureID[cell]
493 end
494
495 # Entrypoints into the interpreter.
496
497 # Expects that CodeBlock is in t1, which is what prologue() leaves behind.
498 macro functionArityCheck(doneLabel, slowPath)
499     loadi PayloadOffset + ArgumentCount[cfr], t0
500     biaeq t0, CodeBlock::m_numParameters[t1], doneLabel
501     prepareStateForCCall()
502     move cfr, a0
503     move PC, a1
504     cCall2(slowPath)   # This slowPath has the protocol: r0 = 0 => no error, r0 != 0 => error
505     btiz r0, .noError
506     move r1, cfr   # r1 contains caller frame
507     jmp _llint_throw_from_slow_path_trampoline
508
509 .noError:
510     loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[r1], t1
511     btiz t1, .continue
512     loadi PayloadOffset + ArgumentCount[cfr], t2
513     addi CallFrameHeaderSlots, t2
514
515     // Check if there are some unaligned slots we can use
516     move t1, t3
517     andi StackAlignmentSlots - 1, t3
518     btiz t3, .noExtraSlot
519     move ValueUndefined, t0
520 .fillExtraSlots:
521     storeq t0, [cfr, t2, 8]
522     addi 1, t2
523     bsubinz 1, t3, .fillExtraSlots
524     andi ~(StackAlignmentSlots - 1), t1
525     btiz t1, .continue
526
527 .noExtraSlot:
528     // Move frame up t1 slots
529     negq t1
530     move cfr, t3
531     subp CalleeSaveSpaceAsVirtualRegisters * 8, t3
532     addi CalleeSaveSpaceAsVirtualRegisters, t2
533 .copyLoop:
534     loadq [t3], t0
535     storeq t0, [t3, t1, 8]
536     addp 8, t3
537     bsubinz 1, t2, .copyLoop
538
539     // Fill new slots with JSUndefined
540     move t1, t2
541     move ValueUndefined, t0
542 .fillLoop:
543     storeq t0, [t3, t1, 8]
544     addp 8, t3
545     baddinz 1, t2, .fillLoop
546
547     lshiftp 3, t1
548     addp t1, cfr
549     addp t1, sp
550
551 .continue:
552     # Reload CodeBlock and reset PC, since the slow_path clobbered them.
553     loadp CodeBlock[cfr], t1
554     loadp CodeBlock::m_instructions[t1], PB
555     move 0, PC
556     jmp doneLabel
557 end
558
559 macro branchIfException(label)
560     loadp Callee[cfr], t3
561     andp MarkedBlockMask, t3
562     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
563     btqz VM::m_exception[t3], .noException
564     jmp label
565 .noException:
566 end
567
568
569 # Instruction implementations
570 _llint_op_enter:
571     traceExecution()
572     checkStackPointerAlignment(t2, 0xdead00e1)
573     loadp CodeBlock[cfr], t2                // t2<CodeBlock> = cfr.CodeBlock
574     loadi CodeBlock::m_numVars[t2], t2      // t2<size_t> = t2<CodeBlock>.m_numVars
575     subq CalleeSaveSpaceAsVirtualRegisters, t2
576     move cfr, t1
577     subq CalleeSaveSpaceAsVirtualRegisters * 8, t1
578     btiz t2, .opEnterDone
579     move ValueUndefined, t0
580     negi t2
581     sxi2q t2, t2
582 .opEnterLoop:
583     storeq t0, [t1, t2, 8]
584     addq 1, t2
585     btqnz t2, .opEnterLoop
586 .opEnterDone:
587     callOpcodeSlowPath(_slow_path_enter)
588     dispatch(1)
589
590
591 _llint_op_argument_count:
592     traceExecution()
593     loadisFromInstruction(1, t1)
594     loadi PayloadOffset + ArgumentCount[cfr], t0
595     subi 1, t0
596     orq TagTypeNumber, t0
597     storeq t0, [cfr, t1, 8]
598     dispatch(2)
599
600
601 _llint_op_get_scope:
602     traceExecution()
603     loadp Callee[cfr], t0
604     loadp JSCallee::m_scope[t0], t0
605     loadisFromInstruction(1, t1)
606     storeq t0, [cfr, t1, 8]
607     dispatch(2)
608
609
610 _llint_op_create_this:
611     traceExecution()
612     loadisFromInstruction(2, t0)
613     loadp [cfr, t0, 8], t0
614     bbneq JSCell::m_type[t0], JSFunctionType, .opCreateThisSlow
615     loadp JSFunction::m_rareData[t0], t3
616     btpz t3, .opCreateThisSlow
617     loadp FunctionRareData::m_objectAllocationProfile + ObjectAllocationProfile::m_allocator[t3], t1
618     loadp FunctionRareData::m_objectAllocationProfile + ObjectAllocationProfile::m_structure[t3], t2
619     btpz t1, .opCreateThisSlow
620     loadpFromInstruction(4, t3)
621     bpeq t3, 1, .hasSeenMultipleCallee
622     bpneq t3, t0, .opCreateThisSlow
623 .hasSeenMultipleCallee:
624     allocateJSObject(t1, t2, t0, t3, .opCreateThisSlow)
625     loadisFromInstruction(1, t1)
626     storeq t0, [cfr, t1, 8]
627     dispatch(5)
628
629 .opCreateThisSlow:
630     callOpcodeSlowPath(_slow_path_create_this)
631     dispatch(5)
632
633
634 _llint_op_to_this:
635     traceExecution()
636     loadisFromInstruction(1, t0)
637     loadq [cfr, t0, 8], t0
638     btqnz t0, tagMask, .opToThisSlow
639     bbneq JSCell::m_type[t0], FinalObjectType, .opToThisSlow
640     loadStructureWithScratch(t0, t1, t2)
641     loadpFromInstruction(2, t2)
642     bpneq t1, t2, .opToThisSlow
643     dispatch(4)
644
645 .opToThisSlow:
646     callOpcodeSlowPath(_slow_path_to_this)
647     dispatch(4)
648
649
650 _llint_op_new_object:
651     traceExecution()
652     loadpFromInstruction(3, t0)
653     loadp ObjectAllocationProfile::m_allocator[t0], t1
654     loadp ObjectAllocationProfile::m_structure[t0], t2
655     allocateJSObject(t1, t2, t0, t3, .opNewObjectSlow)
656     loadisFromInstruction(1, t1)
657     storeq t0, [cfr, t1, 8]
658     dispatch(4)
659
660 .opNewObjectSlow:
661     callOpcodeSlowPath(_llint_slow_path_new_object)
662     dispatch(4)
663
664
665 _llint_op_check_tdz:
666     traceExecution()
667     loadisFromInstruction(1, t0)
668     loadConstantOrVariable(t0, t1)
669     bqneq t1, ValueEmpty, .opNotTDZ
670     callOpcodeSlowPath(_slow_path_throw_tdz_error)
671
672 .opNotTDZ:
673     dispatch(2)
674
675
676 _llint_op_mov:
677     traceExecution()
678     loadisFromInstruction(2, t1)
679     loadisFromInstruction(1, t0)
680     loadConstantOrVariable(t1, t2)
681     storeq t2, [cfr, t0, 8]
682     dispatch(3)
683
684
685 _llint_op_not:
686     traceExecution()
687     loadisFromInstruction(2, t0)
688     loadisFromInstruction(1, t1)
689     loadConstantOrVariable(t0, t2)
690     xorq ValueFalse, t2
691     btqnz t2, ~1, .opNotSlow
692     xorq ValueTrue, t2
693     storeq t2, [cfr, t1, 8]
694     dispatch(3)
695
696 .opNotSlow:
697     callOpcodeSlowPath(_slow_path_not)
698     dispatch(3)
699
700
701 macro equalityComparison(integerComparison, slowPath)
702     traceExecution()
703     loadisFromInstruction(3, t0)
704     loadisFromInstruction(2, t2)
705     loadisFromInstruction(1, t3)
706     loadConstantOrVariableInt32(t0, t1, .slow)
707     loadConstantOrVariableInt32(t2, t0, .slow)
708     integerComparison(t0, t1, t0)
709     orq ValueFalse, t0
710     storeq t0, [cfr, t3, 8]
711     dispatch(4)
712
713 .slow:
714     callOpcodeSlowPath(slowPath)
715     dispatch(4)
716 end
717
718 _llint_op_eq:
719     equalityComparison(
720         macro (left, right, result) cieq left, right, result end,
721         _slow_path_eq)
722
723
724 _llint_op_neq:
725     equalityComparison(
726         macro (left, right, result) cineq left, right, result end,
727         _slow_path_neq)
728
729
730 macro equalNullComparison()
731     loadisFromInstruction(2, t0)
732     loadq [cfr, t0, 8], t0
733     btqnz t0, tagMask, .immediate
734     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
735     move 0, t0
736     jmp .done
737 .masqueradesAsUndefined:
738     loadStructureWithScratch(t0, t2, t1)
739     loadp CodeBlock[cfr], t0
740     loadp CodeBlock::m_globalObject[t0], t0
741     cpeq Structure::m_globalObject[t2], t0, t0
742     jmp .done
743 .immediate:
744     andq ~TagBitUndefined, t0
745     cqeq t0, ValueNull, t0
746 .done:
747 end
748
749 _llint_op_eq_null:
750     traceExecution()
751     equalNullComparison()
752     loadisFromInstruction(1, t1)
753     orq ValueFalse, t0
754     storeq t0, [cfr, t1, 8]
755     dispatch(3)
756
757
758 _llint_op_neq_null:
759     traceExecution()
760     equalNullComparison()
761     loadisFromInstruction(1, t1)
762     xorq ValueTrue, t0
763     storeq t0, [cfr, t1, 8]
764     dispatch(3)
765
766
767 macro strictEq(equalityOperation, slowPath)
768     traceExecution()
769     loadisFromInstruction(3, t0)
770     loadisFromInstruction(2, t2)
771     loadConstantOrVariable(t0, t1)
772     loadConstantOrVariable(t2, t0)
773     move t0, t2
774     orq t1, t2
775     btqz t2, tagMask, .slow
776     bqaeq t0, tagTypeNumber, .leftOK
777     btqnz t0, tagTypeNumber, .slow
778 .leftOK:
779     bqaeq t1, tagTypeNumber, .rightOK
780     btqnz t1, tagTypeNumber, .slow
781 .rightOK:
782     equalityOperation(t0, t1, t0)
783     loadisFromInstruction(1, t1)
784     orq ValueFalse, t0
785     storeq t0, [cfr, t1, 8]
786     dispatch(4)
787
788 .slow:
789     callOpcodeSlowPath(slowPath)
790     dispatch(4)
791 end
792
793 _llint_op_stricteq:
794     strictEq(
795         macro (left, right, result) cqeq left, right, result end,
796         _slow_path_stricteq)
797
798
799 _llint_op_nstricteq:
800     strictEq(
801         macro (left, right, result) cqneq left, right, result end,
802         _slow_path_nstricteq)
803
804
805 macro preOp(arithmeticOperation, slowPath)
806     traceExecution()
807     loadisFromInstruction(1, t0)
808     loadq [cfr, t0, 8], t1
809     bqb t1, tagTypeNumber, .slow
810     arithmeticOperation(t1, .slow)
811     orq tagTypeNumber, t1
812     storeq t1, [cfr, t0, 8]
813     dispatch(2)
814
815 .slow:
816     callOpcodeSlowPath(slowPath)
817     dispatch(2)
818 end
819
820 _llint_op_inc:
821     preOp(
822         macro (value, slow) baddio 1, value, slow end,
823         _slow_path_inc)
824
825
826 _llint_op_dec:
827     preOp(
828         macro (value, slow) bsubio 1, value, slow end,
829         _slow_path_dec)
830
831
832 _llint_op_to_number:
833     traceExecution()
834     loadisFromInstruction(2, t0)
835     loadisFromInstruction(1, t1)
836     loadConstantOrVariable(t0, t2)
837     bqaeq t2, tagTypeNumber, .opToNumberIsImmediate
838     btqz t2, tagTypeNumber, .opToNumberSlow
839 .opToNumberIsImmediate:
840     storeq t2, [cfr, t1, 8]
841     valueProfile(t2, 3, t0)
842     dispatch(4)
843
844 .opToNumberSlow:
845     callOpcodeSlowPath(_slow_path_to_number)
846     dispatch(4)
847
848
849 _llint_op_to_string:
850     traceExecution()
851     loadisFromInstruction(2, t1)
852     loadisFromInstruction(1, t2)
853     loadConstantOrVariable(t1, t0)
854     btqnz t0, tagMask, .opToStringSlow
855     bbneq JSCell::m_type[t0], StringType, .opToStringSlow
856 .opToStringIsString:
857     storeq t0, [cfr, t2, 8]
858     dispatch(3)
859
860 .opToStringSlow:
861     callOpcodeSlowPath(_slow_path_to_string)
862     dispatch(3)
863
864
865 _llint_op_negate:
866     traceExecution()
867     loadisFromInstruction(2, t0)
868     loadisFromInstruction(1, t1)
869     loadConstantOrVariable(t0, t2)
870     bqb t2, tagTypeNumber, .opNegateNotInt
871     btiz t2, 0x7fffffff, .opNegateSlow
872     negi t2
873     orq tagTypeNumber, t2
874     storeq t2, [cfr, t1, 8]
875     dispatch(3)
876 .opNegateNotInt:
877     btqz t2, tagTypeNumber, .opNegateSlow
878     xorq 0x8000000000000000, t2
879     storeq t2, [cfr, t1, 8]
880     dispatch(3)
881
882 .opNegateSlow:
883     callOpcodeSlowPath(_slow_path_negate)
884     dispatch(3)
885
886
887 macro binaryOpCustomStore(integerOperationAndStore, doubleOperation, slowPath)
888     loadisFromInstruction(3, t0)
889     loadisFromInstruction(2, t2)
890     loadConstantOrVariable(t0, t1)
891     loadConstantOrVariable(t2, t0)
892     bqb t0, tagTypeNumber, .op1NotInt
893     bqb t1, tagTypeNumber, .op2NotInt
894     loadisFromInstruction(1, t2)
895     integerOperationAndStore(t1, t0, .slow, t2)
896     dispatch(5)
897
898 .op1NotInt:
899     # First operand is definitely not an int, the second operand could be anything.
900     btqz t0, tagTypeNumber, .slow
901     bqaeq t1, tagTypeNumber, .op1NotIntOp2Int
902     btqz t1, tagTypeNumber, .slow
903     addq tagTypeNumber, t1
904     fq2d t1, ft1
905     jmp .op1NotIntReady
906 .op1NotIntOp2Int:
907     ci2d t1, ft1
908 .op1NotIntReady:
909     loadisFromInstruction(1, t2)
910     addq tagTypeNumber, t0
911     fq2d t0, ft0
912     doubleOperation(ft1, ft0)
913     fd2q ft0, t0
914     subq tagTypeNumber, t0
915     storeq t0, [cfr, t2, 8]
916     dispatch(5)
917
918 .op2NotInt:
919     # First operand is definitely an int, the second is definitely not.
920     loadisFromInstruction(1, t2)
921     btqz t1, tagTypeNumber, .slow
922     ci2d t0, ft0
923     addq tagTypeNumber, t1
924     fq2d t1, ft1
925     doubleOperation(ft1, ft0)
926     fd2q ft0, t0
927     subq tagTypeNumber, t0
928     storeq t0, [cfr, t2, 8]
929     dispatch(5)
930
931 .slow:
932     callOpcodeSlowPath(slowPath)
933     dispatch(5)
934 end
935
936 macro binaryOp(integerOperation, doubleOperation, slowPath)
937     binaryOpCustomStore(
938         macro (left, right, slow, index)
939             integerOperation(left, right, slow)
940             orq tagTypeNumber, right
941             storeq right, [cfr, index, 8]
942         end,
943         doubleOperation, slowPath)
944 end
945
946 _llint_op_add:
947     traceExecution()
948     binaryOp(
949         macro (left, right, slow) baddio left, right, slow end,
950         macro (left, right) addd left, right end,
951         _slow_path_add)
952
953
954 _llint_op_mul:
955     traceExecution()
956     binaryOpCustomStore(
957         macro (left, right, slow, index)
958             # Assume t3 is scratchable.
959             move right, t3
960             bmulio left, t3, slow
961             btinz t3, .done
962             bilt left, 0, slow
963             bilt right, 0, slow
964         .done:
965             orq tagTypeNumber, t3
966             storeq t3, [cfr, index, 8]
967         end,
968         macro (left, right) muld left, right end,
969         _slow_path_mul)
970
971
972 _llint_op_sub:
973     traceExecution()
974     binaryOp(
975         macro (left, right, slow) bsubio left, right, slow end,
976         macro (left, right) subd left, right end,
977         _slow_path_sub)
978
979
980 _llint_op_div:
981     traceExecution()
982     if X86_64 or X86_64_WIN
983         binaryOpCustomStore(
984             macro (left, right, slow, index)
985                 # Assume t3 is scratchable.
986                 btiz left, slow
987                 bineq left, -1, .notNeg2TwoThe31DivByNeg1
988                 bieq right, -2147483648, .slow
989             .notNeg2TwoThe31DivByNeg1:
990                 btinz right, .intOK
991                 bilt left, 0, slow
992             .intOK:
993                 move left, t3
994                 move right, t0
995                 cdqi
996                 idivi t3
997                 btinz t1, slow
998                 orq tagTypeNumber, t0
999                 storeq t0, [cfr, index, 8]
1000             end,
1001             macro (left, right) divd left, right end,
1002             _slow_path_div)
1003     else
1004         callOpcodeSlowPath(_slow_path_div)
1005         dispatch(5)
1006     end
1007
1008
1009 macro bitOp(operation, slowPath, advance)
1010     loadisFromInstruction(3, t0)
1011     loadisFromInstruction(2, t2)
1012     loadisFromInstruction(1, t3)
1013     loadConstantOrVariable(t0, t1)
1014     loadConstantOrVariable(t2, t0)
1015     bqb t0, tagTypeNumber, .slow
1016     bqb t1, tagTypeNumber, .slow
1017     operation(t1, t0)
1018     orq tagTypeNumber, t0
1019     storeq t0, [cfr, t3, 8]
1020     dispatch(advance)
1021
1022 .slow:
1023     callOpcodeSlowPath(slowPath)
1024     dispatch(advance)
1025 end
1026
1027 _llint_op_lshift:
1028     traceExecution()
1029     bitOp(
1030         macro (left, right) lshifti left, right end,
1031         _slow_path_lshift,
1032         4)
1033
1034
1035 _llint_op_rshift:
1036     traceExecution()
1037     bitOp(
1038         macro (left, right) rshifti left, right end,
1039         _slow_path_rshift,
1040         4)
1041
1042
1043 _llint_op_urshift:
1044     traceExecution()
1045     bitOp(
1046         macro (left, right) urshifti left, right end,
1047         _slow_path_urshift,
1048         4)
1049
1050
1051 _llint_op_unsigned:
1052     traceExecution()
1053     loadisFromInstruction(1, t0)
1054     loadisFromInstruction(2, t1)
1055     loadConstantOrVariable(t1, t2)
1056     bilt t2, 0, .opUnsignedSlow
1057     storeq t2, [cfr, t0, 8]
1058     dispatch(3)
1059 .opUnsignedSlow:
1060     callOpcodeSlowPath(_slow_path_unsigned)
1061     dispatch(3)
1062
1063
1064 _llint_op_bitand:
1065     traceExecution()
1066     bitOp(
1067         macro (left, right) andi left, right end,
1068         _slow_path_bitand,
1069         5)
1070
1071
1072 _llint_op_bitxor:
1073     traceExecution()
1074     bitOp(
1075         macro (left, right) xori left, right end,
1076         _slow_path_bitxor,
1077         5)
1078
1079
1080 _llint_op_bitor:
1081     traceExecution()
1082     bitOp(
1083         macro (left, right) ori left, right end,
1084         _slow_path_bitor,
1085         5)
1086
1087
1088 _llint_op_overrides_has_instance:
1089     traceExecution()
1090     loadisFromInstruction(1, t3)
1091
1092     loadisFromInstruction(3, t1)
1093     loadConstantOrVariable(t1, t0)
1094     loadp CodeBlock[cfr], t2
1095     loadp CodeBlock::m_globalObject[t2], t2
1096     loadp JSGlobalObject::m_functionProtoHasInstanceSymbolFunction[t2], t2
1097     bqneq t0, t2, .opOverridesHasInstanceNotDefaultSymbol
1098
1099     loadisFromInstruction(2, t1)
1100     loadConstantOrVariable(t1, t0)
1101     tbz JSCell::m_flags[t0], ImplementsDefaultHasInstance, t1
1102     orq ValueFalse, t1
1103     storeq t1, [cfr, t3, 8]
1104     dispatch(4)
1105
1106 .opOverridesHasInstanceNotDefaultSymbol:
1107     storeq ValueTrue, [cfr, t3, 8]
1108     dispatch(4)
1109
1110
1111 _llint_op_instanceof_custom:
1112     traceExecution()
1113     callOpcodeSlowPath(_llint_slow_path_instanceof_custom)
1114     dispatch(5)
1115
1116
1117 _llint_op_is_empty:
1118     traceExecution()
1119     loadisFromInstruction(2, t1)
1120     loadisFromInstruction(1, t2)
1121     loadConstantOrVariable(t1, t0)
1122     cqeq t0, ValueEmpty, t3
1123     orq ValueFalse, t3
1124     storeq t3, [cfr, t2, 8]
1125     dispatch(3)
1126
1127
1128 _llint_op_is_undefined:
1129     traceExecution()
1130     loadisFromInstruction(2, t1)
1131     loadisFromInstruction(1, t2)
1132     loadConstantOrVariable(t1, t0)
1133     btqz t0, tagMask, .opIsUndefinedCell
1134     cqeq t0, ValueUndefined, t3
1135     orq ValueFalse, t3
1136     storeq t3, [cfr, t2, 8]
1137     dispatch(3)
1138 .opIsUndefinedCell:
1139     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
1140     move ValueFalse, t1
1141     storeq t1, [cfr, t2, 8]
1142     dispatch(3)
1143 .masqueradesAsUndefined:
1144     loadStructureWithScratch(t0, t3, t1)
1145     loadp CodeBlock[cfr], t1
1146     loadp CodeBlock::m_globalObject[t1], t1
1147     cpeq Structure::m_globalObject[t3], t1, t0
1148     orq ValueFalse, t0
1149     storeq t0, [cfr, t2, 8]
1150     dispatch(3)
1151
1152
1153 _llint_op_is_boolean:
1154     traceExecution()
1155     loadisFromInstruction(2, t1)
1156     loadisFromInstruction(1, t2)
1157     loadConstantOrVariable(t1, t0)
1158     xorq ValueFalse, t0
1159     tqz t0, ~1, t0
1160     orq ValueFalse, t0
1161     storeq t0, [cfr, t2, 8]
1162     dispatch(3)
1163
1164
1165 _llint_op_is_number:
1166     traceExecution()
1167     loadisFromInstruction(2, t1)
1168     loadisFromInstruction(1, t2)
1169     loadConstantOrVariable(t1, t0)
1170     tqnz t0, tagTypeNumber, t1
1171     orq ValueFalse, t1
1172     storeq t1, [cfr, t2, 8]
1173     dispatch(3)
1174
1175
1176 _llint_op_is_string:
1177     traceExecution()
1178     loadisFromInstruction(2, t1)
1179     loadisFromInstruction(1, t2)
1180     loadConstantOrVariable(t1, t0)
1181     btqnz t0, tagMask, .opIsStringNotCell
1182     cbeq JSCell::m_type[t0], StringType, t1
1183     orq ValueFalse, t1
1184     storeq t1, [cfr, t2, 8]
1185     dispatch(3)
1186 .opIsStringNotCell:
1187     storeq ValueFalse, [cfr, t2, 8]
1188     dispatch(3)
1189
1190
1191 _llint_op_is_jsarray:
1192     traceExecution()
1193     loadisFromInstruction(2, t1)
1194     loadisFromInstruction(1, t2)
1195     loadConstantOrVariable(t1, t0)
1196     btqnz t0, tagMask, .opIsJSArrayNotCell
1197     cbeq JSCell::m_type[t0], ArrayType, t1
1198     orq ValueFalse, t1
1199     storeq t1, [cfr, t2, 8]
1200     dispatch(3)
1201 .opIsJSArrayNotCell:
1202     storeq ValueFalse, [cfr, t2, 8]
1203     dispatch(3)
1204
1205
1206 _llint_op_is_object:
1207     traceExecution()
1208     loadisFromInstruction(2, t1)
1209     loadisFromInstruction(1, t2)
1210     loadConstantOrVariable(t1, t0)
1211     btqnz t0, tagMask, .opIsObjectNotCell
1212     cbaeq JSCell::m_type[t0], ObjectType, t1
1213     orq ValueFalse, t1
1214     storeq t1, [cfr, t2, 8]
1215     dispatch(3)
1216 .opIsObjectNotCell:
1217     storeq ValueFalse, [cfr, t2, 8]
1218     dispatch(3)
1219
1220
1221 macro loadPropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1222     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1223     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1224     negi propertyOffsetAsInt
1225     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1226     jmp .ready
1227 .isInline:
1228     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1229 .ready:
1230     loadq (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8], value
1231 end
1232
1233
1234 macro storePropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1235     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1236     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1237     negi propertyOffsetAsInt
1238     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1239     jmp .ready
1240 .isInline:
1241     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1242 .ready:
1243     storeq value, (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8]
1244 end
1245
1246 _llint_op_get_by_id:
1247     traceExecution()
1248     loadisFromInstruction(2, t0)
1249     loadConstantOrVariableCell(t0, t3, .opGetByIdSlow)
1250     loadi JSCell::m_structureID[t3], t1
1251     loadisFromInstruction(4, t2)
1252     bineq t2, t1, .opGetByIdSlow
1253     loadisFromInstruction(5, t1)
1254     loadisFromInstruction(1, t2)
1255     loadPropertyAtVariableOffset(t1, t3, t0)
1256     storeq t0, [cfr, t2, 8]
1257     valueProfile(t0, 8, t1)
1258     dispatch(9)
1259
1260 .opGetByIdSlow:
1261     callOpcodeSlowPath(_llint_slow_path_get_by_id)
1262     dispatch(9)
1263
1264
1265 _llint_op_get_by_id_proto_load:
1266     traceExecution()
1267     loadisFromInstruction(2, t0)
1268     loadConstantOrVariableCell(t0, t3, .opGetByIdProtoSlow)
1269     loadi JSCell::m_structureID[t3], t1
1270     loadisFromInstruction(4, t2)
1271     bineq t2, t1, .opGetByIdProtoSlow
1272     loadisFromInstruction(5, t1)
1273     loadpFromInstruction(6, t3)
1274     loadisFromInstruction(1, t2)
1275     loadPropertyAtVariableOffset(t1, t3, t0)
1276     storeq t0, [cfr, t2, 8]
1277     valueProfile(t0, 8, t1)
1278     dispatch(9)
1279
1280 .opGetByIdProtoSlow:
1281     callOpcodeSlowPath(_llint_slow_path_get_by_id)
1282     dispatch(9)
1283
1284
1285 _llint_op_get_by_id_unset:
1286     traceExecution()
1287     loadisFromInstruction(2, t0)
1288     loadConstantOrVariableCell(t0, t3, .opGetByIdUnsetSlow)
1289     loadi JSCell::m_structureID[t3], t1
1290     loadisFromInstruction(4, t2)
1291     bineq t2, t1, .opGetByIdUnsetSlow
1292     loadisFromInstruction(1, t2)
1293     storeq ValueUndefined, [cfr, t2, 8]
1294     valueProfile(ValueUndefined, 8, t1)
1295     dispatch(9)
1296
1297 .opGetByIdUnsetSlow:
1298     callOpcodeSlowPath(_llint_slow_path_get_by_id)
1299     dispatch(9)
1300
1301
1302 _llint_op_get_array_length:
1303     traceExecution()
1304     loadisFromInstruction(2, t0)
1305     loadpFromInstruction(4, t1)
1306     loadConstantOrVariableCell(t0, t3, .opGetArrayLengthSlow)
1307     move t3, t2
1308     arrayProfile(t2, t1, t0)
1309     btiz t2, IsArray, .opGetArrayLengthSlow
1310     btiz t2, IndexingShapeMask, .opGetArrayLengthSlow
1311     loadisFromInstruction(1, t1)
1312     loadp JSObject::m_butterfly[t3], t0
1313     loadi -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], t0
1314     bilt t0, 0, .opGetArrayLengthSlow
1315     orq tagTypeNumber, t0
1316     valueProfile(t0, 8, t2)
1317     storeq t0, [cfr, t1, 8]
1318     dispatch(9)
1319
1320 .opGetArrayLengthSlow:
1321     callOpcodeSlowPath(_llint_slow_path_get_by_id)
1322     dispatch(9)
1323
1324
1325 _llint_op_put_by_id:
1326     traceExecution()
1327     writeBarrierOnOperands(1, 3)
1328     loadisFromInstruction(1, t3)
1329     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
1330     loadisFromInstruction(4, t2)
1331     bineq t2, JSCell::m_structureID[t0], .opPutByIdSlow
1332
1333     # At this point, we have:
1334     # t2 -> current structure ID
1335     # t0 -> object base
1336
1337     loadisFromInstruction(3, t1)
1338     loadConstantOrVariable(t1, t3)
1339
1340     loadpFromInstruction(8, t1)
1341
1342     # At this point, we have:
1343     # t0 -> object base
1344     # t1 -> put by id flags
1345     # t2 -> current structure ID
1346     # t3 -> value to put
1347
1348     btpnz t1, PutByIdPrimaryTypeMask, .opPutByIdTypeCheckObjectWithStructureOrOther
1349
1350     # We have one of the non-structure type checks. Find out which one.
1351     andp PutByIdSecondaryTypeMask, t1
1352     bplt t1, PutByIdSecondaryTypeString, .opPutByIdTypeCheckLessThanString
1353
1354     # We are one of the following: String, Symbol, Object, ObjectOrOther, Top
1355     bplt t1, PutByIdSecondaryTypeObjectOrOther, .opPutByIdTypeCheckLessThanObjectOrOther
1356
1357     # We are either ObjectOrOther or Top.
1358     bpeq t1, PutByIdSecondaryTypeTop, .opPutByIdDoneCheckingTypes
1359
1360     # Check if we are ObjectOrOther.
1361     btqz t3, tagMask, .opPutByIdTypeCheckObject
1362 .opPutByIdTypeCheckOther:
1363     andq ~TagBitUndefined, t3
1364     bqeq t3, ValueNull, .opPutByIdDoneCheckingTypes
1365     jmp .opPutByIdSlow
1366
1367 .opPutByIdTypeCheckLessThanObjectOrOther:
1368     # We are either String, Symbol or Object.
1369     btqnz t3, tagMask, .opPutByIdSlow
1370     bpeq t1, PutByIdSecondaryTypeObject, .opPutByIdTypeCheckObject
1371     bpeq t1, PutByIdSecondaryTypeSymbol, .opPutByIdTypeCheckSymbol
1372     bbeq JSCell::m_type[t3], StringType, .opPutByIdDoneCheckingTypes
1373     jmp .opPutByIdSlow
1374 .opPutByIdTypeCheckObject:
1375     bbaeq JSCell::m_type[t3], ObjectType, .opPutByIdDoneCheckingTypes
1376     jmp .opPutByIdSlow
1377 .opPutByIdTypeCheckSymbol:
1378     bbeq JSCell::m_type[t3], SymbolType, .opPutByIdDoneCheckingTypes
1379     jmp .opPutByIdSlow
1380
1381 .opPutByIdTypeCheckLessThanString:
1382     # We are one of the following: Bottom, Boolean, Other, Int32, Number
1383     bplt t1, PutByIdSecondaryTypeInt32, .opPutByIdTypeCheckLessThanInt32
1384
1385     # We are either Int32 or Number.
1386     bpeq t1, PutByIdSecondaryTypeNumber, .opPutByIdTypeCheckNumber
1387
1388     bqaeq t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1389     jmp .opPutByIdSlow
1390
1391 .opPutByIdTypeCheckNumber:
1392     btqnz t3, tagTypeNumber, .opPutByIdDoneCheckingTypes
1393     jmp .opPutByIdSlow
1394
1395 .opPutByIdTypeCheckLessThanInt32:
1396     # We are one of the following: Bottom, Boolean, Other.
1397     bpneq t1, PutByIdSecondaryTypeBoolean, .opPutByIdTypeCheckBottomOrOther
1398     xorq ValueFalse, t3
1399     btqz t3, ~1, .opPutByIdDoneCheckingTypes
1400     jmp .opPutByIdSlow
1401
1402 .opPutByIdTypeCheckBottomOrOther:
1403     bpeq t1, PutByIdSecondaryTypeOther, .opPutByIdTypeCheckOther
1404     jmp .opPutByIdSlow
1405
1406 .opPutByIdTypeCheckObjectWithStructureOrOther:
1407     btqz t3, tagMask, .opPutByIdTypeCheckObjectWithStructure
1408     btpnz t1, PutByIdPrimaryTypeObjectWithStructureOrOther, .opPutByIdTypeCheckOther
1409     jmp .opPutByIdSlow
1410
1411 .opPutByIdTypeCheckObjectWithStructure:
1412     urshiftp 3, t1
1413     bineq t1, JSCell::m_structureID[t3], .opPutByIdSlow
1414
1415 .opPutByIdDoneCheckingTypes:
1416     loadisFromInstruction(6, t1)
1417     
1418     btiz t1, .opPutByIdNotTransition
1419
1420     # This is the transition case. t1 holds the new structureID. t2 holds the old structure ID.
1421     # If we have a chain, we need to check it. t0 is the base. We may clobber t1 to use it as
1422     # scratch.
1423     loadpFromInstruction(7, t3)
1424     btpz t3, .opPutByIdTransitionDirect
1425
1426     loadp StructureChain::m_vector[t3], t3
1427     assert(macro (ok) btpnz t3, ok end)
1428
1429     structureIDToStructureWithScratch(t2, t1)
1430     loadq Structure::m_prototype[t2], t2
1431     bqeq t2, ValueNull, .opPutByIdTransitionChainDone
1432 .opPutByIdTransitionChainLoop:
1433     # At this point, t2 contains a prototye, and [t3] contains the Structure* that we want that
1434     # prototype to have. We don't want to have to load the Structure* for t2. Instead, we load
1435     # the Structure* from [t3], and then we compare its id to the id in the header of t2.
1436     loadp [t3], t1
1437     loadi JSCell::m_structureID[t2], t2
1438     # Now, t1 has the Structure* and t2 has the StructureID that we want that Structure* to have.
1439     bineq t2, Structure::m_blob + StructureIDBlob::u.fields.structureID[t1], .opPutByIdSlow
1440     addp 8, t3
1441     loadq Structure::m_prototype[t1], t2
1442     bqneq t2, ValueNull, .opPutByIdTransitionChainLoop
1443
1444 .opPutByIdTransitionChainDone:
1445     # Reload the new structure, since we clobbered it above.
1446     loadisFromInstruction(6, t1)
1447
1448 .opPutByIdTransitionDirect:
1449     storei t1, JSCell::m_structureID[t0]
1450
1451 .opPutByIdNotTransition:
1452     # The only thing live right now is t0, which holds the base.
1453     loadisFromInstruction(3, t1)
1454     loadConstantOrVariable(t1, t2)
1455     loadisFromInstruction(5, t1)
1456     storePropertyAtVariableOffset(t1, t0, t2)
1457     dispatch(9)
1458
1459 .opPutByIdSlow:
1460     callOpcodeSlowPath(_llint_slow_path_put_by_id)
1461     dispatch(9)
1462
1463 macro finishGetByVal(result, scratch)
1464     loadisFromInstruction(1, scratch)
1465     storeq result, [cfr, scratch, 8]
1466     valueProfile(result, 5, scratch)
1467     dispatch(6)
1468 end
1469
1470 macro finishIntGetByVal(result, scratch)
1471     orq tagTypeNumber, result
1472     finishGetByVal(result, scratch)
1473 end
1474
1475 macro finishDoubleGetByVal(result, scratch1, scratch2)
1476     fd2q result, scratch1
1477     subq tagTypeNumber, scratch1
1478     finishGetByVal(scratch1, scratch2)
1479 end
1480
1481 _llint_op_get_by_val:
1482     traceExecution()
1483     loadisFromInstruction(2, t2)
1484     loadConstantOrVariableCell(t2, t0, .opGetByValSlow)
1485     loadpFromInstruction(4, t3)
1486     move t0, t2
1487     arrayProfile(t2, t3, t1)
1488     loadisFromInstruction(3, t3)
1489     loadConstantOrVariableInt32(t3, t1, .opGetByValSlow)
1490     sxi2q t1, t1
1491     loadp JSObject::m_butterfly[t0], t3
1492     andi IndexingShapeMask, t2
1493     bieq t2, Int32Shape, .opGetByValIsContiguous
1494     bineq t2, ContiguousShape, .opGetByValNotContiguous
1495 .opGetByValIsContiguous:
1496
1497     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1498     loadisFromInstruction(1, t0)
1499     loadq [t3, t1, 8], t2
1500     btqz t2, .opGetByValOutOfBounds
1501     jmp .opGetByValDone
1502
1503 .opGetByValNotContiguous:
1504     bineq t2, DoubleShape, .opGetByValNotDouble
1505     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1506     loadis 8[PB, PC, 8], t0
1507     loadd [t3, t1, 8], ft0
1508     bdnequn ft0, ft0, .opGetByValOutOfBounds
1509     fd2q ft0, t2
1510     subq tagTypeNumber, t2
1511     jmp .opGetByValDone
1512     
1513 .opGetByValNotDouble:
1514     subi ArrayStorageShape, t2
1515     bia t2, SlowPutArrayStorageShape - ArrayStorageShape, .opGetByValNotIndexedStorage
1516     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t3], .opGetByValOutOfBounds
1517     loadisFromInstruction(1, t0)
1518     loadq ArrayStorage::m_vector[t3, t1, 8], t2
1519     btqz t2, .opGetByValOutOfBounds
1520
1521 .opGetByValDone:
1522     storeq t2, [cfr, t0, 8]
1523     valueProfile(t2, 5, t0)
1524     dispatch(6)
1525
1526 .opGetByValOutOfBounds:
1527     loadpFromInstruction(4, t0)
1528     storeb 1, ArrayProfile::m_outOfBounds[t0]
1529
1530 .opGetByValNotIndexedStorage:
1531     # First lets check if we even have a typed array. This lets us do some boilerplate up front.
1532     loadb JSCell::m_type[t0], t2
1533     subi FirstArrayType, t2
1534     bia t2, LastArrayType - FirstArrayType, .opGetByValSlow
1535     
1536     # Sweet, now we know that we have a typed array. Do some basic things now.
1537     loadp JSArrayBufferView::m_vector[t0], t3
1538     biaeq t1, JSArrayBufferView::m_length[t0], .opGetByValSlow
1539     
1540     # Now bisect through the various types. Note that we can treat Uint8ArrayType and
1541     # Uint8ClampedArrayType the same.
1542     bia t2, Uint8ClampedArrayType - FirstArrayType, .opGetByValAboveUint8ClampedArray
1543     
1544     # We have one of Int8ArrayType .. Uint8ClampedArrayType.
1545     bia t2, Int16ArrayType - FirstArrayType, .opGetByValInt32ArrayOrUint8Array
1546     
1547     # We have one of Int8ArrayType or Int16ArrayType
1548     bineq t2, Int8ArrayType - FirstArrayType, .opGetByValInt16Array
1549     
1550     # We have Int8ArrayType
1551     loadbs [t3, t1], t0
1552     finishIntGetByVal(t0, t1)
1553
1554 .opGetByValInt16Array:
1555     loadhs [t3, t1, 2], t0
1556     finishIntGetByVal(t0, t1)
1557
1558 .opGetByValInt32ArrayOrUint8Array:
1559     # We have one of Int16Array, Uint8Array, or Uint8ClampedArray.
1560     bieq t2, Int32ArrayType - FirstArrayType, .opGetByValInt32Array
1561     
1562     # We have either Uint8Array or Uint8ClampedArray. They behave the same so that's cool.
1563     loadb [t3, t1], t0
1564     finishIntGetByVal(t0, t1)
1565
1566 .opGetByValInt32Array:
1567     loadi [t3, t1, 4], t0
1568     finishIntGetByVal(t0, t1)
1569
1570 .opGetByValAboveUint8ClampedArray:
1571     # We have one of Uint16ArrayType .. Float64ArrayType.
1572     bia t2, Uint32ArrayType - FirstArrayType, .opGetByValAboveUint32Array
1573     
1574     # We have either Uint16ArrayType or Uint32ArrayType.
1575     bieq t2, Uint32ArrayType - FirstArrayType, .opGetByValUint32Array
1576
1577     # We have Uint16ArrayType.
1578     loadh [t3, t1, 2], t0
1579     finishIntGetByVal(t0, t1)
1580
1581 .opGetByValUint32Array:
1582     # This is the hardest part because of large unsigned values.
1583     loadi [t3, t1, 4], t0
1584     bilt t0, 0, .opGetByValSlow # This case is still awkward to implement in LLInt.
1585     finishIntGetByVal(t0, t1)
1586
1587 .opGetByValAboveUint32Array:
1588     # We have one of Float32ArrayType or Float64ArrayType. Sadly, we cannot handle Float32Array
1589     # inline yet. That would require some offlineasm changes.
1590     bieq t2, Float32ArrayType - FirstArrayType, .opGetByValSlow
1591
1592     # We have Float64ArrayType.
1593     loadd [t3, t1, 8], ft0
1594     bdnequn ft0, ft0, .opGetByValSlow
1595     finishDoubleGetByVal(ft0, t0, t1)
1596
1597 .opGetByValSlow:
1598     callOpcodeSlowPath(_llint_slow_path_get_by_val)
1599     dispatch(6)
1600
1601
1602 macro contiguousPutByVal(storeCallback)
1603     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .outOfBounds
1604 .storeResult:
1605     loadisFromInstruction(3, t2)
1606     storeCallback(t2, t1, [t0, t3, 8])
1607     dispatch(5)
1608
1609 .outOfBounds:
1610     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1611     loadp 32[PB, PC, 8], t2
1612     storeb 1, ArrayProfile::m_mayStoreToHole[t2]
1613     addi 1, t3, t2
1614     storei t2, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1615     jmp .storeResult
1616 end
1617
1618 macro putByVal(slowPath)
1619     traceExecution()
1620     writeBarrierOnOperands(1, 3)
1621     loadisFromInstruction(1, t0)
1622     loadConstantOrVariableCell(t0, t1, .opPutByValSlow)
1623     loadpFromInstruction(4, t3)
1624     move t1, t2
1625     arrayProfile(t2, t3, t0)
1626     loadisFromInstruction(2, t0)
1627     loadConstantOrVariableInt32(t0, t3, .opPutByValSlow)
1628     sxi2q t3, t3
1629     loadp JSObject::m_butterfly[t1], t0
1630     andi IndexingShapeMask, t2
1631     bineq t2, Int32Shape, .opPutByValNotInt32
1632     contiguousPutByVal(
1633         macro (operand, scratch, address)
1634             loadConstantOrVariable(operand, scratch)
1635             bpb scratch, tagTypeNumber, .opPutByValSlow
1636             storep scratch, address
1637         end)
1638
1639 .opPutByValNotInt32:
1640     bineq t2, DoubleShape, .opPutByValNotDouble
1641     contiguousPutByVal(
1642         macro (operand, scratch, address)
1643             loadConstantOrVariable(operand, scratch)
1644             bqb scratch, tagTypeNumber, .notInt
1645             ci2d scratch, ft0
1646             jmp .ready
1647         .notInt:
1648             addp tagTypeNumber, scratch
1649             fq2d scratch, ft0
1650             bdnequn ft0, ft0, .opPutByValSlow
1651         .ready:
1652             stored ft0, address
1653         end)
1654
1655 .opPutByValNotDouble:
1656     bineq t2, ContiguousShape, .opPutByValNotContiguous
1657     contiguousPutByVal(
1658         macro (operand, scratch, address)
1659             loadConstantOrVariable(operand, scratch)
1660             storep scratch, address
1661         end)
1662
1663 .opPutByValNotContiguous:
1664     bineq t2, ArrayStorageShape, .opPutByValSlow
1665     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1666     btqz ArrayStorage::m_vector[t0, t3, 8], .opPutByValArrayStorageEmpty
1667 .opPutByValArrayStorageStoreResult:
1668     loadisFromInstruction(3, t2)
1669     loadConstantOrVariable(t2, t1)
1670     storeq t1, ArrayStorage::m_vector[t0, t3, 8]
1671     dispatch(5)
1672
1673 .opPutByValArrayStorageEmpty:
1674     loadpFromInstruction(4, t1)
1675     storeb 1, ArrayProfile::m_mayStoreToHole[t1]
1676     addi 1, ArrayStorage::m_numValuesInVector[t0]
1677     bib t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .opPutByValArrayStorageStoreResult
1678     addi 1, t3, t1
1679     storei t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1680     jmp .opPutByValArrayStorageStoreResult
1681
1682 .opPutByValOutOfBounds:
1683     loadpFromInstruction(4, t0)
1684     storeb 1, ArrayProfile::m_outOfBounds[t0]
1685 .opPutByValSlow:
1686     callOpcodeSlowPath(slowPath)
1687     dispatch(5)
1688 end
1689
1690 _llint_op_put_by_val:
1691     putByVal(_llint_slow_path_put_by_val)
1692
1693 _llint_op_put_by_val_direct:
1694     putByVal(_llint_slow_path_put_by_val_direct)
1695
1696
1697 _llint_op_jmp:
1698     traceExecution()
1699     dispatchIntIndirect(1)
1700
1701
1702 macro jumpTrueOrFalse(conditionOp, slow)
1703     loadisFromInstruction(1, t1)
1704     loadConstantOrVariable(t1, t0)
1705     xorq ValueFalse, t0
1706     btqnz t0, -1, .slow
1707     conditionOp(t0, .target)
1708     dispatch(3)
1709
1710 .target:
1711     dispatchIntIndirect(2)
1712
1713 .slow:
1714     callOpcodeSlowPath(slow)
1715     dispatch(0)
1716 end
1717
1718
1719 macro equalNull(cellHandler, immediateHandler)
1720     loadisFromInstruction(1, t0)
1721     assertNotConstant(t0)
1722     loadq [cfr, t0, 8], t0
1723     btqnz t0, tagMask, .immediate
1724     loadStructureWithScratch(t0, t2, t1)
1725     cellHandler(t2, JSCell::m_flags[t0], .target)
1726     dispatch(3)
1727
1728 .target:
1729     dispatchIntIndirect(2)
1730
1731 .immediate:
1732     andq ~TagBitUndefined, t0
1733     immediateHandler(t0, .target)
1734     dispatch(3)
1735 end
1736
1737 _llint_op_jeq_null:
1738     traceExecution()
1739     equalNull(
1740         macro (structure, value, target) 
1741             btbz value, MasqueradesAsUndefined, .notMasqueradesAsUndefined
1742             loadp CodeBlock[cfr], t0
1743             loadp CodeBlock::m_globalObject[t0], t0
1744             bpeq Structure::m_globalObject[structure], t0, target
1745 .notMasqueradesAsUndefined:
1746         end,
1747         macro (value, target) bqeq value, ValueNull, target end)
1748
1749
1750 _llint_op_jneq_null:
1751     traceExecution()
1752     equalNull(
1753         macro (structure, value, target) 
1754             btbz value, MasqueradesAsUndefined, target
1755             loadp CodeBlock[cfr], t0
1756             loadp CodeBlock::m_globalObject[t0], t0
1757             bpneq Structure::m_globalObject[structure], t0, target
1758         end,
1759         macro (value, target) bqneq value, ValueNull, target end)
1760
1761
1762 _llint_op_jneq_ptr:
1763     traceExecution()
1764     loadisFromInstruction(1, t0)
1765     loadisFromInstruction(2, t1)
1766     loadp CodeBlock[cfr], t2
1767     loadp CodeBlock::m_globalObject[t2], t2
1768     loadp JSGlobalObject::m_specialPointers[t2, t1, 8], t1
1769     bpneq t1, [cfr, t0, 8], .opJneqPtrTarget
1770     dispatch(4)
1771
1772 .opJneqPtrTarget:
1773     dispatchIntIndirect(3)
1774
1775
1776 macro compare(integerCompare, doubleCompare, slowPath)
1777     loadisFromInstruction(1, t2)
1778     loadisFromInstruction(2, t3)
1779     loadConstantOrVariable(t2, t0)
1780     loadConstantOrVariable(t3, t1)
1781     bqb t0, tagTypeNumber, .op1NotInt
1782     bqb t1, tagTypeNumber, .op2NotInt
1783     integerCompare(t0, t1, .jumpTarget)
1784     dispatch(4)
1785
1786 .op1NotInt:
1787     btqz t0, tagTypeNumber, .slow
1788     bqb t1, tagTypeNumber, .op1NotIntOp2NotInt
1789     ci2d t1, ft1
1790     jmp .op1NotIntReady
1791 .op1NotIntOp2NotInt:
1792     btqz t1, tagTypeNumber, .slow
1793     addq tagTypeNumber, t1
1794     fq2d t1, ft1
1795 .op1NotIntReady:
1796     addq tagTypeNumber, t0
1797     fq2d t0, ft0
1798     doubleCompare(ft0, ft1, .jumpTarget)
1799     dispatch(4)
1800
1801 .op2NotInt:
1802     ci2d t0, ft0
1803     btqz t1, tagTypeNumber, .slow
1804     addq tagTypeNumber, t1
1805     fq2d t1, ft1
1806     doubleCompare(ft0, ft1, .jumpTarget)
1807     dispatch(4)
1808
1809 .jumpTarget:
1810     dispatchIntIndirect(3)
1811
1812 .slow:
1813     callOpcodeSlowPath(slowPath)
1814     dispatch(0)
1815 end
1816
1817
1818 _llint_op_switch_imm:
1819     traceExecution()
1820     loadisFromInstruction(3, t2)
1821     loadisFromInstruction(1, t3)
1822     loadConstantOrVariable(t2, t1)
1823     loadp CodeBlock[cfr], t2
1824     loadp CodeBlock::m_rareData[t2], t2
1825     muli sizeof SimpleJumpTable, t3    # FIXME: would be nice to peephole this!
1826     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1827     addp t3, t2
1828     bqb t1, tagTypeNumber, .opSwitchImmNotInt
1829     subi SimpleJumpTable::min[t2], t1
1830     biaeq t1, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchImmFallThrough
1831     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t3
1832     loadis [t3, t1, 4], t1
1833     btiz t1, .opSwitchImmFallThrough
1834     dispatch(t1)
1835
1836 .opSwitchImmNotInt:
1837     btqnz t1, tagTypeNumber, .opSwitchImmSlow   # Go slow if it's a double.
1838 .opSwitchImmFallThrough:
1839     dispatchIntIndirect(2)
1840
1841 .opSwitchImmSlow:
1842     callOpcodeSlowPath(_llint_slow_path_switch_imm)
1843     dispatch(0)
1844
1845
1846 _llint_op_switch_char:
1847     traceExecution()
1848     loadisFromInstruction(3, t2)
1849     loadisFromInstruction(1, t3)
1850     loadConstantOrVariable(t2, t1)
1851     loadp CodeBlock[cfr], t2
1852     loadp CodeBlock::m_rareData[t2], t2
1853     muli sizeof SimpleJumpTable, t3
1854     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1855     addp t3, t2
1856     btqnz t1, tagMask, .opSwitchCharFallThrough
1857     bbneq JSCell::m_type[t1], StringType, .opSwitchCharFallThrough
1858     bineq JSString::m_length[t1], 1, .opSwitchCharFallThrough
1859     loadp JSString::m_value[t1], t0
1860     btpz  t0, .opSwitchOnRope
1861     loadp StringImpl::m_data8[t0], t1
1862     btinz StringImpl::m_hashAndFlags[t0], HashFlags8BitBuffer, .opSwitchChar8Bit
1863     loadh [t1], t0
1864     jmp .opSwitchCharReady
1865 .opSwitchChar8Bit:
1866     loadb [t1], t0
1867 .opSwitchCharReady:
1868     subi SimpleJumpTable::min[t2], t0
1869     biaeq t0, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchCharFallThrough
1870     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t2
1871     loadis [t2, t0, 4], t1
1872     btiz t1, .opSwitchCharFallThrough
1873     dispatch(t1)
1874
1875 .opSwitchCharFallThrough:
1876     dispatchIntIndirect(2)
1877
1878 .opSwitchOnRope:
1879     callOpcodeSlowPath(_llint_slow_path_switch_char)
1880     dispatch(0)
1881
1882
1883 macro arrayProfileForCall()
1884     loadisFromInstruction(4, t3)
1885     negp t3
1886     loadq ThisArgumentOffset[cfr, t3, 8], t0
1887     btqnz t0, tagMask, .done
1888     loadpFromInstruction((CallOpCodeSize - 2), t1)
1889     loadi JSCell::m_structureID[t0], t3
1890     storei t3, ArrayProfile::m_lastSeenStructureID[t1]
1891 .done:
1892 end
1893
1894 macro doCall(slowPath, prepareCall)
1895     loadisFromInstruction(2, t0)
1896     loadpFromInstruction(5, t1)
1897     loadp LLIntCallLinkInfo::callee[t1], t2
1898     loadConstantOrVariable(t0, t3)
1899     bqneq t3, t2, .opCallSlow
1900     loadisFromInstruction(4, t3)
1901     lshifti 3, t3
1902     negp t3
1903     addp cfr, t3
1904     storeq t2, Callee[t3]
1905     loadisFromInstruction(3, t2)
1906     storei PC, ArgumentCount + TagOffset[cfr]
1907     storei t2, ArgumentCount + PayloadOffset[t3]
1908     move t3, sp
1909     prepareCall(LLIntCallLinkInfo::machineCodeTarget[t1], t2, t3, t4)
1910     callTargetFunction(LLIntCallLinkInfo::machineCodeTarget[t1])
1911
1912 .opCallSlow:
1913     slowPathForCall(slowPath, prepareCall)
1914 end
1915
1916 _llint_op_ret:
1917     traceExecution()
1918     checkSwitchToJITForEpilogue()
1919     loadisFromInstruction(1, t2)
1920     loadConstantOrVariable(t2, r0)
1921     doReturn()
1922
1923
1924 _llint_op_to_primitive:
1925     traceExecution()
1926     loadisFromInstruction(2, t2)
1927     loadisFromInstruction(1, t3)
1928     loadConstantOrVariable(t2, t0)
1929     btqnz t0, tagMask, .opToPrimitiveIsImm
1930     bbaeq JSCell::m_type[t0], ObjectType, .opToPrimitiveSlowCase
1931 .opToPrimitiveIsImm:
1932     storeq t0, [cfr, t3, 8]
1933     dispatch(3)
1934
1935 .opToPrimitiveSlowCase:
1936     callOpcodeSlowPath(_slow_path_to_primitive)
1937     dispatch(3)
1938
1939
1940 _llint_op_catch:
1941     # This is where we end up from the JIT's throw trampoline (because the
1942     # machine code return address will be set to _llint_op_catch), and from
1943     # the interpreter's throw trampoline (see _llint_throw_trampoline).
1944     # The throwing code must have known that we were throwing to the interpreter,
1945     # and have set VM::targetInterpreterPCForThrow.
1946     loadp Callee[cfr], t3
1947     andp MarkedBlockMask, t3
1948     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1949     restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(t3, t0)
1950     loadp VM::callFrameForCatch[t3], cfr
1951     storep 0, VM::callFrameForCatch[t3]
1952     restoreStackPointerAfterCall()
1953
1954     loadp CodeBlock[cfr], PB
1955     loadp CodeBlock::m_instructions[PB], PB
1956     loadp VM::targetInterpreterPCForThrow[t3], PC
1957     subp PB, PC
1958     rshiftp 3, PC
1959
1960     callOpcodeSlowPath(_llint_slow_path_check_if_exception_is_uncatchable_and_notify_profiler)
1961     bpeq r1, 0, .isCatchableException
1962     jmp _llint_throw_from_slow_path_trampoline
1963
1964 .isCatchableException:
1965     loadp Callee[cfr], t3
1966     andp MarkedBlockMask, t3
1967     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1968
1969     loadq VM::m_exception[t3], t0
1970     storeq 0, VM::m_exception[t3]
1971     loadisFromInstruction(1, t2)
1972     storeq t0, [cfr, t2, 8]
1973
1974     loadq Exception::m_value[t0], t3
1975     loadisFromInstruction(2, t2)
1976     storeq t3, [cfr, t2, 8]
1977
1978     traceExecution()
1979     dispatch(3)
1980
1981
1982 _llint_op_end:
1983     traceExecution()
1984     checkSwitchToJITForEpilogue()
1985     loadisFromInstruction(1, t0)
1986     assertNotConstant(t0)
1987     loadq [cfr, t0, 8], r0
1988     doReturn()
1989
1990
1991 _llint_throw_from_slow_path_trampoline:
1992     loadp Callee[cfr], t1
1993     andp MarkedBlockMask, t1
1994     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
1995     copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(t1, t2)
1996
1997     callSlowPath(_llint_slow_path_handle_exception)
1998
1999     # When throwing from the interpreter (i.e. throwing from LLIntSlowPaths), so
2000     # the throw target is not necessarily interpreted code, we come to here.
2001     # This essentially emulates the JIT's throwing protocol.
2002     loadp Callee[cfr], t1
2003     andp MarkedBlockMask, t1
2004     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
2005     jmp VM::targetMachinePCForThrow[t1]
2006
2007
2008 _llint_throw_during_call_trampoline:
2009     preserveReturnAddressAfterCall(t2)
2010     jmp _llint_throw_from_slow_path_trampoline
2011
2012
2013 macro nativeCallTrampoline(executableOffsetToFunction)
2014
2015     functionPrologue()
2016     storep 0, CodeBlock[cfr]
2017     loadp Callee[cfr], t0
2018     andp MarkedBlockMask, t0, t1
2019     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t1], t1
2020     storep cfr, VM::topCallFrame[t1]
2021     if ARM64 or C_LOOP
2022         storep lr, ReturnPC[cfr]
2023     end
2024     move cfr, a0
2025     loadp Callee[cfr], t1
2026     loadp JSFunction::m_executable[t1], t1
2027     checkStackPointerAlignment(t3, 0xdead0001)
2028     if C_LOOP
2029         cloopCallNative executableOffsetToFunction[t1]
2030     else
2031         if X86_64_WIN
2032             subp 32, sp
2033         end
2034         call executableOffsetToFunction[t1]
2035         if X86_64_WIN
2036             addp 32, sp
2037         end
2038     end
2039     loadp Callee[cfr], t3
2040     andp MarkedBlockMask, t3
2041     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
2042
2043     functionEpilogue()
2044
2045     btqnz VM::m_exception[t3], .handleException
2046     ret
2047
2048 .handleException:
2049     storep cfr, VM::topCallFrame[t3]
2050     restoreStackPointerAfterCall()
2051     jmp _llint_throw_from_slow_path_trampoline
2052 end
2053
2054 macro getConstantScope(dst)
2055     loadpFromInstruction(6, t0)
2056     loadisFromInstruction(dst, t1)
2057     storeq t0, [cfr, t1, 8]
2058 end
2059
2060 macro varInjectionCheck(slowPath)
2061     loadp CodeBlock[cfr], t0
2062     loadp CodeBlock::m_globalObject[t0], t0
2063     loadp JSGlobalObject::m_varInjectionWatchpoint[t0], t0
2064     bbeq WatchpointSet::m_state[t0], IsInvalidated, slowPath
2065 end
2066
2067 macro resolveScope()
2068     loadisFromInstruction(5, t2)
2069     loadisFromInstruction(2, t0)
2070     loadp [cfr, t0, 8], t0
2071     btiz t2, .resolveScopeLoopEnd
2072
2073 .resolveScopeLoop:
2074     loadp JSScope::m_next[t0], t0
2075     subi 1, t2
2076     btinz t2, .resolveScopeLoop
2077
2078 .resolveScopeLoopEnd:
2079     loadisFromInstruction(1, t1)
2080     storeq t0, [cfr, t1, 8]
2081 end
2082
2083
2084 _llint_op_resolve_scope:
2085     traceExecution()
2086     loadisFromInstruction(4, t0)
2087
2088 #rGlobalProperty:
2089     bineq t0, GlobalProperty, .rGlobalVar
2090     getConstantScope(1)
2091     dispatch(7)
2092
2093 .rGlobalVar:
2094     bineq t0, GlobalVar, .rGlobalLexicalVar
2095     getConstantScope(1)
2096     dispatch(7)
2097
2098 .rGlobalLexicalVar:
2099     bineq t0, GlobalLexicalVar, .rClosureVar
2100     getConstantScope(1)
2101     dispatch(7)
2102
2103 .rClosureVar:
2104     bineq t0, ClosureVar, .rModuleVar
2105     resolveScope()
2106     dispatch(7)
2107
2108 .rModuleVar:
2109     bineq t0, ModuleVar, .rGlobalPropertyWithVarInjectionChecks
2110     getConstantScope(1)
2111     dispatch(7)
2112
2113 .rGlobalPropertyWithVarInjectionChecks:
2114     bineq t0, GlobalPropertyWithVarInjectionChecks, .rGlobalVarWithVarInjectionChecks
2115     varInjectionCheck(.rDynamic)
2116     getConstantScope(1)
2117     dispatch(7)
2118
2119 .rGlobalVarWithVarInjectionChecks:
2120     bineq t0, GlobalVarWithVarInjectionChecks, .rGlobalLexicalVarWithVarInjectionChecks
2121     varInjectionCheck(.rDynamic)
2122     getConstantScope(1)
2123     dispatch(7)
2124
2125 .rGlobalLexicalVarWithVarInjectionChecks:
2126     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .rClosureVarWithVarInjectionChecks
2127     varInjectionCheck(.rDynamic)
2128     getConstantScope(1)
2129     dispatch(7)
2130
2131 .rClosureVarWithVarInjectionChecks:
2132     bineq t0, ClosureVarWithVarInjectionChecks, .rDynamic
2133     varInjectionCheck(.rDynamic)
2134     resolveScope()
2135     dispatch(7)
2136
2137 .rDynamic:
2138     callOpcodeSlowPath(_slow_path_resolve_scope)
2139     dispatch(7)
2140
2141
2142 macro loadWithStructureCheck(operand, slowPath)
2143     loadisFromInstruction(operand, t0)
2144     loadq [cfr, t0, 8], t0
2145     loadStructureWithScratch(t0, t2, t1)
2146     loadpFromInstruction(5, t1)
2147     bpneq t2, t1, slowPath
2148 end
2149
2150 macro getProperty()
2151     loadisFromInstruction(6, t1)
2152     loadPropertyAtVariableOffset(t1, t0, t2)
2153     valueProfile(t2, 7, t0)
2154     loadisFromInstruction(1, t0)
2155     storeq t2, [cfr, t0, 8]
2156 end
2157
2158 macro getGlobalVar(tdzCheckIfNecessary)
2159     loadpFromInstruction(6, t0)
2160     loadq [t0], t0
2161     tdzCheckIfNecessary(t0)
2162     valueProfile(t0, 7, t1)
2163     loadisFromInstruction(1, t1)
2164     storeq t0, [cfr, t1, 8]
2165 end
2166
2167 macro getClosureVar()
2168     loadisFromInstruction(6, t1)
2169     loadq JSEnvironmentRecord_variables[t0, t1, 8], t0
2170     valueProfile(t0, 7, t1)
2171     loadisFromInstruction(1, t1)
2172     storeq t0, [cfr, t1, 8]
2173 end
2174
2175 _llint_op_get_from_scope:
2176     traceExecution()
2177     loadisFromInstruction(4, t0)
2178     andi ResolveTypeMask, t0
2179
2180 #gGlobalProperty:
2181     bineq t0, GlobalProperty, .gGlobalVar
2182     loadWithStructureCheck(2, .gDynamic)
2183     getProperty()
2184     dispatch(8)
2185
2186 .gGlobalVar:
2187     bineq t0, GlobalVar, .gGlobalLexicalVar
2188     getGlobalVar(macro(v) end)
2189     dispatch(8)
2190
2191 .gGlobalLexicalVar:
2192     bineq t0, GlobalLexicalVar, .gClosureVar
2193     getGlobalVar(
2194         macro (value)
2195             bqeq value, ValueEmpty, .gDynamic
2196         end)
2197     dispatch(8)
2198
2199 .gClosureVar:
2200     bineq t0, ClosureVar, .gGlobalPropertyWithVarInjectionChecks
2201     loadVariable(2, t0)
2202     getClosureVar()
2203     dispatch(8)
2204
2205 .gGlobalPropertyWithVarInjectionChecks:
2206     bineq t0, GlobalPropertyWithVarInjectionChecks, .gGlobalVarWithVarInjectionChecks
2207     loadWithStructureCheck(2, .gDynamic)
2208     getProperty()
2209     dispatch(8)
2210
2211 .gGlobalVarWithVarInjectionChecks:
2212     bineq t0, GlobalVarWithVarInjectionChecks, .gGlobalLexicalVarWithVarInjectionChecks
2213     varInjectionCheck(.gDynamic)
2214     getGlobalVar(macro(v) end)
2215     dispatch(8)
2216
2217 .gGlobalLexicalVarWithVarInjectionChecks:
2218     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .gClosureVarWithVarInjectionChecks
2219     varInjectionCheck(.gDynamic)
2220     getGlobalVar(
2221         macro (value)
2222             bqeq value, ValueEmpty, .gDynamic
2223         end)
2224     dispatch(8)
2225
2226 .gClosureVarWithVarInjectionChecks:
2227     bineq t0, ClosureVarWithVarInjectionChecks, .gDynamic
2228     varInjectionCheck(.gDynamic)
2229     loadVariable(2, t0)
2230     getClosureVar()
2231     dispatch(8)
2232
2233 .gDynamic:
2234     callOpcodeSlowPath(_llint_slow_path_get_from_scope)
2235     dispatch(8)
2236
2237
2238 macro putProperty()
2239     loadisFromInstruction(3, t1)
2240     loadConstantOrVariable(t1, t2)
2241     loadisFromInstruction(6, t1)
2242     storePropertyAtVariableOffset(t1, t0, t2)
2243 end
2244
2245 macro putGlobalVariable()
2246     loadisFromInstruction(3, t0)
2247     loadConstantOrVariable(t0, t1)
2248     loadpFromInstruction(5, t2)
2249     loadpFromInstruction(6, t0)
2250     notifyWrite(t2, .pDynamic)
2251     storeq t1, [t0]
2252 end
2253
2254 macro putClosureVar()
2255     loadisFromInstruction(3, t1)
2256     loadConstantOrVariable(t1, t2)
2257     loadisFromInstruction(6, t1)
2258     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2259 end
2260
2261 macro putLocalClosureVar()
2262     loadisFromInstruction(3, t1)
2263     loadConstantOrVariable(t1, t2)
2264     loadpFromInstruction(5, t3)
2265     btpz t3, .noVariableWatchpointSet
2266     notifyWrite(t3, .pDynamic)
2267 .noVariableWatchpointSet:
2268     loadisFromInstruction(6, t1)
2269     storeq t2, JSEnvironmentRecord_variables[t0, t1, 8]
2270 end
2271
2272 macro checkTDZInGlobalPutToScopeIfNecessary()
2273     loadisFromInstruction(4, t0)
2274     andi InitializationModeMask, t0
2275     rshifti InitializationModeShift, t0
2276     bineq t0, NotInitialization, .noNeedForTDZCheck
2277     loadpFromInstruction(6, t0)
2278     loadq [t0], t0
2279     bqeq t0, ValueEmpty, .pDynamic
2280 .noNeedForTDZCheck:
2281 end
2282
2283
2284 _llint_op_put_to_scope:
2285     traceExecution()
2286     loadisFromInstruction(4, t0)
2287     andi ResolveTypeMask, t0
2288
2289 #pLocalClosureVar:
2290     bineq t0, LocalClosureVar, .pGlobalProperty
2291     writeBarrierOnOperands(1, 3)
2292     loadVariable(1, t0)
2293     putLocalClosureVar()
2294     dispatch(7)
2295
2296 .pGlobalProperty:
2297     bineq t0, GlobalProperty, .pGlobalVar
2298     writeBarrierOnOperands(1, 3)
2299     loadWithStructureCheck(1, .pDynamic)
2300     putProperty()
2301     dispatch(7)
2302
2303 .pGlobalVar:
2304     bineq t0, GlobalVar, .pGlobalLexicalVar
2305     writeBarrierOnGlobalObject(3)
2306     putGlobalVariable()
2307     dispatch(7)
2308
2309 .pGlobalLexicalVar:
2310     bineq t0, GlobalLexicalVar, .pClosureVar
2311     writeBarrierOnGlobalLexicalEnvironment(3)
2312     checkTDZInGlobalPutToScopeIfNecessary()
2313     putGlobalVariable()
2314     dispatch(7)
2315
2316 .pClosureVar:
2317     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
2318     writeBarrierOnOperands(1, 3)
2319     loadVariable(1, t0)
2320     putClosureVar()
2321     dispatch(7)
2322
2323 .pGlobalPropertyWithVarInjectionChecks:
2324     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
2325     writeBarrierOnOperands(1, 3)
2326     loadWithStructureCheck(1, .pDynamic)
2327     putProperty()
2328     dispatch(7)
2329
2330 .pGlobalVarWithVarInjectionChecks:
2331     bineq t0, GlobalVarWithVarInjectionChecks, .pGlobalLexicalVarWithVarInjectionChecks
2332     writeBarrierOnGlobalObject(3)
2333     varInjectionCheck(.pDynamic)
2334     putGlobalVariable()
2335     dispatch(7)
2336
2337 .pGlobalLexicalVarWithVarInjectionChecks:
2338     bineq t0, GlobalLexicalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
2339     writeBarrierOnGlobalLexicalEnvironment(3)
2340     varInjectionCheck(.pDynamic)
2341     checkTDZInGlobalPutToScopeIfNecessary()
2342     putGlobalVariable()
2343     dispatch(7)
2344
2345 .pClosureVarWithVarInjectionChecks:
2346     bineq t0, ClosureVarWithVarInjectionChecks, .pModuleVar
2347     writeBarrierOnOperands(1, 3)
2348     varInjectionCheck(.pDynamic)
2349     loadVariable(1, t0)
2350     putClosureVar()
2351     dispatch(7)
2352
2353 .pModuleVar:
2354     bineq t0, ModuleVar, .pDynamic
2355     callOpcodeSlowPath(_slow_path_throw_strict_mode_readonly_property_write_error)
2356     dispatch(7)
2357
2358 .pDynamic:
2359     callOpcodeSlowPath(_llint_slow_path_put_to_scope)
2360     dispatch(7)
2361
2362
2363 _llint_op_get_from_arguments:
2364     traceExecution()
2365     loadVariable(2, t0)
2366     loadi 24[PB, PC, 8], t1
2367     loadq DirectArguments_storage[t0, t1, 8], t0
2368     valueProfile(t0, 4, t1)
2369     loadisFromInstruction(1, t1)
2370     storeq t0, [cfr, t1, 8]
2371     dispatch(5)
2372
2373
2374 _llint_op_put_to_arguments:
2375     traceExecution()
2376     writeBarrierOnOperands(1, 3)
2377     loadVariable(1, t0)
2378     loadi 16[PB, PC, 8], t1
2379     loadisFromInstruction(3, t3)
2380     loadConstantOrVariable(t3, t2)
2381     storeq t2, DirectArguments_storage[t0, t1, 8]
2382     dispatch(4)
2383
2384
2385 _llint_op_get_parent_scope:
2386     traceExecution()
2387     loadVariable(2, t0)
2388     loadp JSScope::m_next[t0], t0
2389     loadisFromInstruction(1, t1)
2390     storeq t0, [cfr, t1, 8]
2391     dispatch(3)
2392
2393
2394 _llint_op_profile_type:
2395     traceExecution()
2396     loadp CodeBlock[cfr], t1
2397     loadp CodeBlock::m_vm[t1], t1
2398     # t1 is holding the pointer to the typeProfilerLog.
2399     loadp VM::m_typeProfilerLog[t1], t1
2400     # t2 is holding the pointer to the current log entry.
2401     loadp TypeProfilerLog::m_currentLogEntryPtr[t1], t2
2402
2403     # t0 is holding the JSValue argument.
2404     loadisFromInstruction(1, t3)
2405     loadConstantOrVariable(t3, t0)
2406
2407     bqeq t0, ValueEmpty, .opProfileTypeDone
2408     # Store the JSValue onto the log entry.
2409     storeq t0, TypeProfilerLog::LogEntry::value[t2]
2410     
2411     # Store the TypeLocation onto the log entry.
2412     loadpFromInstruction(2, t3)
2413     storep t3, TypeProfilerLog::LogEntry::location[t2]
2414
2415     btqz t0, tagMask, .opProfileTypeIsCell
2416     storei 0, TypeProfilerLog::LogEntry::structureID[t2]
2417     jmp .opProfileTypeSkipIsCell
2418 .opProfileTypeIsCell:
2419     loadi JSCell::m_structureID[t0], t3
2420     storei t3, TypeProfilerLog::LogEntry::structureID[t2]
2421 .opProfileTypeSkipIsCell:
2422     
2423     # Increment the current log entry.
2424     addp sizeof TypeProfilerLog::LogEntry, t2
2425     storep t2, TypeProfilerLog::m_currentLogEntryPtr[t1]
2426
2427     loadp TypeProfilerLog::m_logEndPtr[t1], t1
2428     bpneq t2, t1, .opProfileTypeDone
2429     callOpcodeSlowPath(_slow_path_profile_type_clear_log)
2430
2431 .opProfileTypeDone:
2432     dispatch(6)
2433
2434 _llint_op_profile_control_flow:
2435     traceExecution()
2436     loadpFromInstruction(1, t0)
2437     addq 1, BasicBlockLocation::m_executionCount[t0]
2438     dispatch(2)
2439
2440
2441 _llint_op_get_rest_length:
2442     traceExecution()
2443     loadi PayloadOffset + ArgumentCount[cfr], t0
2444     subi 1, t0
2445     loadisFromInstruction(2, t1)
2446     bilteq t0, t1, .storeZero
2447     subi t1, t0
2448     jmp .boxUp
2449 .storeZero:
2450     move 0, t0
2451 .boxUp:
2452     orq tagTypeNumber, t0
2453     loadisFromInstruction(1, t1)
2454     storeq t0, [cfr, t1, 8]
2455     dispatch(3)
2456
2457
2458 _llint_op_log_shadow_chicken_prologue:
2459     traceExecution()
2460     acquireShadowChickenPacket(.opLogShadowChickenPrologueSlow)
2461     storep cfr, ShadowChicken::Packet::frame[t0]
2462     loadp CallerFrame[cfr], t1
2463     storep t1, ShadowChicken::Packet::callerFrame[t0]
2464     loadp Callee[cfr], t1
2465     storep t1, ShadowChicken::Packet::callee[t0]
2466     loadVariable(1, t1)
2467     storep t1, ShadowChicken::Packet::scope[t0]
2468     dispatch(2)
2469 .opLogShadowChickenPrologueSlow:
2470     callOpcodeSlowPath(_llint_slow_path_log_shadow_chicken_prologue)
2471     dispatch(2)
2472
2473
2474 _llint_op_log_shadow_chicken_tail:
2475     traceExecution()
2476     acquireShadowChickenPacket(.opLogShadowChickenTailSlow)
2477     storep cfr, ShadowChicken::Packet::frame[t0]
2478     storep ShadowChickenTailMarker, ShadowChicken::Packet::callee[t0]
2479     loadVariable(1, t1)
2480     storep t1, ShadowChicken::Packet::thisValue[t0]
2481     loadVariable(2, t1)
2482     storep t1, ShadowChicken::Packet::scope[t0]
2483     loadp CodeBlock[cfr], t1
2484     storep t1, ShadowChicken::Packet::codeBlock[t0]
2485     storei PC, ShadowChicken::Packet::callSiteIndex[t0]
2486     dispatch(3)
2487 .opLogShadowChickenTailSlow:
2488     callOpcodeSlowPath(_llint_slow_path_log_shadow_chicken_tail)
2489     dispatch(3)