Add write barriers to the LLInt
authormhahnenberg@apple.com <mhahnenberg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 6 Jan 2014 23:04:25 +0000 (23:04 +0000)
committermhahnenberg@apple.com <mhahnenberg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 6 Jan 2014 23:04:25 +0000 (23:04 +0000)
https://bugs.webkit.org/show_bug.cgi?id=126527

Reviewed by Filip Pizlo.

This patch takes a similar approach to how write barriers work in the baseline JIT.
We execute the write barrier at the beginning of the opcode so we don't have to
worry about saving and restoring live registers across write barrier slow path calls
to C code.

* llint/LLIntOfflineAsmConfig.h:
* llint/LLIntSlowPaths.cpp:
(JSC::LLInt::llint_write_barrier_slow):
* llint/LLIntSlowPaths.h:
* llint/LowLevelInterpreter.asm:
* llint/LowLevelInterpreter32_64.asm:
* llint/LowLevelInterpreter64.asm:
* offlineasm/arm64.rb:
* offlineasm/instructions.rb:
* offlineasm/x86.rb:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@161377 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/JavaScriptCore/ChangeLog
Source/JavaScriptCore/llint/LLIntOfflineAsmConfig.h
Source/JavaScriptCore/llint/LLIntSlowPaths.cpp
Source/JavaScriptCore/llint/LLIntSlowPaths.h
Source/JavaScriptCore/llint/LowLevelInterpreter.asm
Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm
Source/JavaScriptCore/llint/LowLevelInterpreter64.asm
Source/JavaScriptCore/offlineasm/arm.rb
Source/JavaScriptCore/offlineasm/arm64.rb
Source/JavaScriptCore/offlineasm/mips.rb
Source/JavaScriptCore/offlineasm/x86.rb

index 762e143..6e8d469 100644 (file)
@@ -1,3 +1,26 @@
+2014-01-06  Mark Hahnenberg  <mhahnenberg@apple.com>
+
+        Add write barriers to the LLInt
+        https://bugs.webkit.org/show_bug.cgi?id=126527
+
+        Reviewed by Filip Pizlo.
+
+        This patch takes a similar approach to how write barriers work in the baseline JIT.
+        We execute the write barrier at the beginning of the opcode so we don't have to 
+        worry about saving and restoring live registers across write barrier slow path calls 
+        to C code.
+
+        * llint/LLIntOfflineAsmConfig.h:
+        * llint/LLIntSlowPaths.cpp:
+        (JSC::LLInt::llint_write_barrier_slow):
+        * llint/LLIntSlowPaths.h:
+        * llint/LowLevelInterpreter.asm:
+        * llint/LowLevelInterpreter32_64.asm:
+        * llint/LowLevelInterpreter64.asm:
+        * offlineasm/arm64.rb:
+        * offlineasm/instructions.rb:
+        * offlineasm/x86.rb:
+
 2014-01-05  Sam Weinig  <sam@webkit.org>
 
         [JS] Implement Promise.all()
index 423069d..07a91bb 100644 (file)
 #define OFFLINE_ASM_ALWAYS_ALLOCATE_SLOW 0
 #endif
 
+#if ENABLE(GGC)
+#define OFFLINE_ASM_GGC 1
+#else
+#define OFFLINE_ASM_GGC 0
+#endif
+
 #endif // LLIntOfflineAsmConfig_h
index 4c7f489..2626417 100644 (file)
@@ -1388,6 +1388,11 @@ LLINT_SLOW_PATH_DECL(slow_path_put_to_scope)
     LLINT_END();
 }
 
+extern "C" void llint_write_barrier_slow(ExecState*, JSCell* cell)
+{
+    Heap::writeBarrier(cell);
+}
+
 } } // namespace JSC::LLInt
 
 #endif // ENABLE(LLINT)
index 789dc59..8d60afa 100644 (file)
@@ -41,6 +41,7 @@ namespace LLInt {
 
 extern "C" SlowPathReturnType llint_trace_operand(ExecState*, Instruction*, int fromWhere, int operand);
 extern "C" SlowPathReturnType llint_trace_value(ExecState*, Instruction*, int fromWhere, int operand);
+extern "C" void llint_write_barrier_slow(ExecState*, JSCell*);
 
 #define LLINT_SLOW_PATH_DECL(name) \
     extern "C" SlowPathReturnType llint_##name(ExecState* exec, Instruction* pc)
index 5c1612d..396551b 100644 (file)
@@ -170,7 +170,11 @@ const Dynamic = 6
 
 const ResolveModeMask = 0xffff
 
-const MarkedBlockMask = ~0xffff
+const MarkedBlockSize = 64 * 1024
+const MarkedBlockMask = ~(MarkedBlockSize - 1)
+# Constants for checking mark bits.
+const AtomNumberShift = 3
+const BitMapWordShift = 4
 
 # Allocation constants
 if JSVALUE64
@@ -263,6 +267,18 @@ macro arrayProfile(structureAndIndexingType, profile, scratch)
     loadb Structure::m_indexingType[structure], indexingType
 end
 
+macro checkMarkByte(cell, scratch1, scratch2, continuation)
+    move cell, scratch1
+    move cell, scratch2
+
+    andp MarkedBlockMask, scratch1
+    andp ~MarkedBlockMask, scratch2
+
+    rshiftp AtomNumberShift + BitMapWordShift, scratch2
+    loadb MarkedBlock::m_marks[scratch1, scratch2, 1], scratch1
+    continuation(scratch1)
+end
+
 macro checkSwitchToJIT(increment, action)
     loadp CodeBlock[cfr], t0
     baddis increment, CodeBlock::m_llintExecuteCounter + ExecutionCounter::m_counter[t0], .continue
index f498448..ae1f3d2 100644 (file)
@@ -491,8 +491,56 @@ macro loadConstantOrVariablePayloadUnchecked(index, payload)
         payload)
 end
 
-macro writeBarrier(tag, payload)
-    # Nothing to do, since we don't have a generational or incremental collector.
+macro writeBarrierOnOperand(cellOperand)
+    if GGC
+        loadisFromInstruction(cellOperand, t1)
+        loadConstantOrVariablePayload(t1, CellTag, t0, .writeBarrierDone)
+        checkMarkByte(t0, t1, t2, 
+            macro(marked)
+                btbz marked, .writeBarrierDone
+                push cfr, PC
+                # We make two extra slots because cCall2 will poke.
+                subp 8, sp
+                cCall2(_llint_write_barrier_slow, cfr, t0)
+                addp 8, sp
+                pop PC, cfr
+            end
+        )
+    .writeBarrierDone:
+    end
+end
+
+macro writeBarrierOnOperands(cellOperand, valueOperand)
+    if GGC
+        loadisFromInstruction(valueOperand, t1)
+        loadConstantOrVariableTag(t1, t0)
+        bineq t0, CellTag, .writeBarrierDone
+    
+        writeBarrierOnOperand(cellOperand)
+    .writeBarrierDone:
+    end
+end
+
+macro writeBarrierOnGlobalObject(valueOperand)
+    if GGC
+        loadisFromInstruction(valueOperand, t1)
+        bineq t0, CellTag, .writeBarrierDone
+    
+        loadp CodeBlock[cfr], t0
+        loadp CodeBlock::m_globalObject[t0], t0
+        checkMarkByte(t0, t1, t2,
+            macro(marked)
+                btbz marked, .writeBarrierDone
+                push cfr, PC
+                # We make two extra slots because cCall2 will poke.
+                subp 8, sp
+                cCall2(_llint_write_barrier_slow, cfr, t0)
+                addp 8, sp
+                pop PC, cfr
+            end
+        )
+    .writeBarrierDone:
+    end
 end
 
 macro valueProfile(tag, payload, operand, scratch)
@@ -575,6 +623,7 @@ _llint_op_enter:
     addi 1, t2
     btinz t2, .opEnterLoop
 .opEnterDone:
+    callSlowPath(_slow_path_enter)
     dispatch(1)
 
 
@@ -1255,10 +1304,10 @@ end
 
 _llint_op_init_global_const:
     traceExecution()
+    writeBarrierOnGlobalObject(2)
     loadi 8[PC], t1
     loadi 4[PC], t0
     loadConstantOrVariable(t1, t2, t3)
-    writeBarrier(t2, t3)
     storei t2, TagOffset[t0]
     storei t3, PayloadOffset[t0]
     dispatch(5)
@@ -1344,6 +1393,7 @@ _llint_op_get_arguments_length:
 
 macro putById(getPropertyStorage)
     traceExecution()
+    writeBarrierOnOperands(1, 3)
     loadi 4[PC], t3
     loadi 16[PC], t1
     loadConstantOrVariablePayload(t3, CellTag, t0, .opPutByIdSlow)
@@ -1355,7 +1405,6 @@ macro putById(getPropertyStorage)
             bpneq JSCell::m_structure[t0], t1, .opPutByIdSlow
             loadi 20[PC], t1
             loadConstantOrVariable2Reg(t2, scratch, t2)
-            writeBarrier(scratch, t2)
             storei scratch, TagOffset[propertyStorage, t1]
             storei t2, PayloadOffset[propertyStorage, t1]
             dispatch(9)
@@ -1376,6 +1425,7 @@ _llint_op_put_by_id_out_of_line:
 
 macro putByIdTransition(additionalChecks, getPropertyStorage)
     traceExecution()
+    writeBarrierOnOperand(1)
     loadi 4[PC], t3
     loadi 16[PC], t1
     loadConstantOrVariablePayload(t3, CellTag, t0, .opPutByIdSlow)
@@ -1389,7 +1439,6 @@ macro putByIdTransition(additionalChecks, getPropertyStorage)
         macro (propertyStorage, scratch)
             addp t1, propertyStorage, t3
             loadConstantOrVariable2Reg(t2, t1, t2)
-            writeBarrier(t1, t2)
             storei t1, TagOffset[t3]
             loadi 24[PC], t1
             storei t2, PayloadOffset[t3]
@@ -1561,6 +1610,7 @@ end
 
 macro putByVal(holeCheck, slowPath)
     traceExecution()
+    writeBarrierOnOperands(1, 3)
     loadi 4[PC], t0
     loadConstantOrVariablePayload(t0, CellTag, t1, .opPutByValSlow)
     loadp JSCell::m_structure[t1], t2
@@ -1602,7 +1652,6 @@ macro putByVal(holeCheck, slowPath)
             const tag = scratch
             const payload = operand
             loadConstantOrVariable2Reg(operand, tag, payload)
-            writeBarrier(tag, payload)
             storei tag, TagOffset[base, index, 8]
             storei payload, PayloadOffset[base, index, 8]
         end)
@@ -1614,7 +1663,6 @@ macro putByVal(holeCheck, slowPath)
 .opPutByValArrayStorageStoreResult:
     loadi 12[PC], t2
     loadConstantOrVariable2Reg(t2, t1, t2)
-    writeBarrier(t1, t2)
     storei t1, ArrayStorage::m_vector + TagOffset[t0, t3, 8]
     storei t2, ArrayStorage::m_vector + PayloadOffset[t0, t3, 8]
     dispatch(5)
@@ -2339,35 +2387,41 @@ _llint_op_put_to_scope:
 
 #pGlobalProperty:
     bineq t0, GlobalProperty, .pGlobalVar
+    writeBarrierOnOperands(1, 3)
     loadWithStructureCheck(1, .pDynamic)
     putProperty()
     dispatch(7)
 
 .pGlobalVar:
     bineq t0, GlobalVar, .pClosureVar
+    writeBarrierOnGlobalObject(3)
     putGlobalVar()
     dispatch(7)
 
 .pClosureVar:
     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
+    writeBarrierOnOperands(1, 3)
     loadVariable(1, t2, t1, t0)
     putClosureVar()
     dispatch(7)
 
 .pGlobalPropertyWithVarInjectionChecks:
     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
+    writeBarrierOnOperands(1, 3)
     loadWithStructureCheck(1, .pDynamic)
     putProperty()
     dispatch(7)
 
 .pGlobalVarWithVarInjectionChecks:
     bineq t0, GlobalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
+    writeBarrierOnGlobalObject(3)
     varInjectionCheck(.pDynamic)
     putGlobalVar()
     dispatch(7)
 
 .pClosureVarWithVarInjectionChecks:
     bineq t0, ClosureVarWithVarInjectionChecks, .pDynamic
+    writeBarrierOnOperands(1, 3)
     varInjectionCheck(.pDynamic)
     loadVariable(1, t2, t1, t0)
     putClosureVar()
index c92156c..16da960 100644 (file)
@@ -331,8 +331,51 @@ macro loadConstantOrVariableCell(index, value, slow)
     btqnz value, tagMask, slow
 end
 
-macro writeBarrier(value)
-    # Nothing to do, since we don't have a generational or incremental collector.
+macro writeBarrierOnOperand(cellOperand)
+    if GGC
+        loadisFromInstruction(cellOperand, t1)
+        loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
+        checkMarkByte(t0, t1, t2, 
+            macro(marked)
+                btbz marked, .writeBarrierDone
+                push PB, PC
+                cCall2(_llint_write_barrier_slow, cfr, t0)
+                push PC, PB
+            end
+        )
+    .writeBarrierDone:
+    end
+end
+
+macro writeBarrierOnOperands(cellOperand, valueOperand)
+    if GGC
+        loadisFromInstruction(valueOperand, t1)
+        loadConstantOrVariable(t1, t0)
+        btpz t0, .writeBarrierDone
+    
+        writeBarrierOnOperand(cellOperand)
+    .writeBarrierDone:
+    end
+end
+
+macro writeBarrierOnGlobalObject(valueOperand)
+    if GGC
+        loadisFromInstruction(valueOperand, t1)
+        loadConstantOrVariable(t1, t0)
+        btpz t0, .writeBarrierDone
+    
+        loadp CodeBlock[cfr], t0
+        loadp CodeBlock::m_globalObject[t0], t0
+        checkMarkByte(t0, t1, t2,
+            macro(marked)
+                btbz marked, .writeBarrierDone
+                push PB, PC
+                cCall2(_llint_write_barrier_slow, cfr, t0)
+                pop PC, PB
+            end
+        )
+    .writeBarrierDone:
+    end
 end
 
 macro valueProfile(value, operand, scratch)
@@ -412,6 +455,7 @@ _llint_op_enter:
     addq 1, t2
     btqnz t2, .opEnterLoop
 .opEnterDone:
+    callSlowPath(_slow_path_enter)
     dispatch(1)
 
 
@@ -1064,10 +1108,10 @@ end
 
 _llint_op_init_global_const:
     traceExecution()
+    writeBarrierOnGlobalObject(2)
     loadisFromInstruction(2, t1)
     loadpFromInstruction(1, t0)
     loadConstantOrVariable(t1, t2)
-    writeBarrier(t2)
     storeq t2, [t0]
     dispatch(5)
 
@@ -1149,6 +1193,7 @@ _llint_op_get_arguments_length:
 
 macro putById(getPropertyStorage)
     traceExecution()
+    writeBarrierOnOperands(1, 3)
     loadisFromInstruction(1, t3)
     loadpFromInstruction(4, t1)
     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
@@ -1160,7 +1205,6 @@ macro putById(getPropertyStorage)
             bpneq JSCell::m_structure[t0], t1, .opPutByIdSlow
             loadisFromInstruction(5, t1)
             loadConstantOrVariable(t2, scratch)
-            writeBarrier(t0)
             storeq scratch, [propertyStorage, t1]
             dispatch(9)
         end)
@@ -1180,6 +1224,7 @@ _llint_op_put_by_id_out_of_line:
 
 macro putByIdTransition(additionalChecks, getPropertyStorage)
     traceExecution()
+    writeBarrierOnOperand(1)
     loadisFromInstruction(1, t3)
     loadpFromInstruction(4, t1)
     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
@@ -1193,7 +1238,6 @@ macro putByIdTransition(additionalChecks, getPropertyStorage)
         macro (propertyStorage, scratch)
             addp t1, propertyStorage, t3
             loadConstantOrVariable(t2, t1)
-            writeBarrier(t1)
             storeq t1, [t3]
             loadpFromInstruction(6, t1)
             storep t1, JSCell::m_structure[t0]
@@ -1362,6 +1406,7 @@ end
 
 macro putByVal(holeCheck, slowPath)
     traceExecution()
+    writeBarrierOnOperands(1, 3)
     loadisFromInstruction(1, t0)
     loadConstantOrVariableCell(t0, t1, .opPutByValSlow)
     loadp JSCell::m_structure[t1], t2
@@ -1401,7 +1446,6 @@ macro putByVal(holeCheck, slowPath)
     contiguousPutByVal(
         macro (operand, scratch, address)
             loadConstantOrVariable(operand, scratch)
-            writeBarrier(scratch)
             storep scratch, address
         end)
 
@@ -1412,7 +1456,6 @@ macro putByVal(holeCheck, slowPath)
 .opPutByValArrayStorageStoreResult:
     loadisFromInstruction(3, t2)
     loadConstantOrVariable(t2, t1)
-    writeBarrier(t1)
     storeq t1, ArrayStorage::m_vector[t0, t3, 8]
     dispatch(5)
 
@@ -2107,35 +2150,41 @@ _llint_op_put_to_scope:
 
 #pGlobalProperty:
     bineq t0, GlobalProperty, .pGlobalVar
+    writeBarrierOnOperands(1, 3)
     loadWithStructureCheck(1, .pDynamic)
     putProperty()
     dispatch(7)
 
 .pGlobalVar:
     bineq t0, GlobalVar, .pClosureVar
+    writeBarrierOnGlobalObject(3)
     putGlobalVar()
     dispatch(7)
 
 .pClosureVar:
     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
+    writeBarrierOnOperands(1, 3)
     loadVariable(1, t0)
     putClosureVar()
     dispatch(7)
 
 .pGlobalPropertyWithVarInjectionChecks:
     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
+    writeBarrierOnOperands(1, 3)
     loadWithStructureCheck(1, .pDynamic)
     putProperty()
     dispatch(7)
 
 .pGlobalVarWithVarInjectionChecks:
     bineq t0, GlobalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
+    writeBarrierOnGlobalObject(3)
     varInjectionCheck(.pDynamic)
     putGlobalVar()
     dispatch(7)
 
 .pClosureVarWithVarInjectionChecks:
     bineq t0, ClosureVarWithVarInjectionChecks, .pDynamic
+    writeBarrierOnOperands(1, 3)
     varInjectionCheck(.pDynamic)
     loadVariable(1, t0)
     putClosureVar()
index a36676a..7de75ea 100644 (file)
@@ -457,9 +457,15 @@ class Instruction
             # FIXME: either support this or remove it.
             raise "ARM does not support this opcode yet, #{codeOrigin}"
         when "pop"
-            $asm.puts "pop { #{operands[0].armOperand} }"
+            operands.each {
+                | op |
+                $asm.puts "pop { #{op.armOperand} }"
+            }
         when "push"
-            $asm.puts "push { #{operands[0].armOperand} }"
+            operands.each {
+                | op |
+                $asm.puts "push { #{op.armOperand} }"
+            }
         when "popCalleeSaves"
             if isARMv7
                 $asm.puts "pop {r4-r6, r8-r11}"                
index 1cc3666..e0a23ff 100644 (file)
@@ -566,11 +566,24 @@ class Instruction
             # FIXME: Remove it or support it.
             raise "ARM64 does not support this opcode yet, #{codeOriginString}"
         when "pop"
-            # FIXME: Remove it or support it.
-            raise "ARM64 does not support this opcode yet, #{codeOriginString}"
+            operands.each_slice(2) {
+                | ops |
+                # Note that the operands are in the reverse order of the case for push.
+                # This is due to the fact that order matters for pushing and popping, and 
+                # on platforms that only push/pop one slot at a time they pop their 
+                # arguments in the reverse order that they were pushed. In order to remain 
+                # compatible with those platforms we assume here that that's what has been done.
+
+                # So for example, if we did push(A, B, C, D), we would then pop(D, C, B, A).
+                # But since the ordering of arguments doesn't change on arm64 between the stp and ldp 
+                # instructions we need to flip flop the argument positions that were passed to us.
+                $asm.puts "ldp #{ops[1].arm64Operand(:ptr)}, #{ops[0].arm64Operand(:ptr)}, [sp], #16"
+            }
         when "push"
-            # FIXME: Remove it or support it.
-            raise "ARM64 does not support this opcode yet, #{codeOriginString}"
+            operands.each_slice(2) {
+                | ops |
+                $asm.puts "stp #{ops[0].arm64Operand(:ptr)}, #{ops[1].arm64Operand(:ptr)}, [sp, #-16]!"
+            }
         when "popLRAndFP"
             $asm.puts "ldp fp, lr, [sp], #16"
         when "pushLRAndFP"
index 8fc96be..686f58f 100644 (file)
@@ -839,11 +839,17 @@ class Instruction
             # FIXME: either support this or remove it.
             raise "MIPS does not support this opcode yet, #{codeOrigin}"
         when "pop"
-            $asm.puts "lw #{operands[0].mipsOperand}, 0($sp)"
-            $asm.puts "addiu $sp, $sp, 4"
+            operands.each {
+                | op |
+                $asm.puts "lw #{op.mipsOperand}, 0($sp)"
+                $asm.puts "addiu $sp, $sp, 4"
+            }
         when "push"
-            $asm.puts "addiu $sp, $sp, -4"
-            $asm.puts "sw #{operands[0].mipsOperand}, 0($sp)"
+            operands.each {
+                | op |
+                $asm.puts "addiu $sp, $sp, -4"
+                $asm.puts "sw #{op.mipsOperand}, 0($sp)"
+            }
         when "popCalleeSaves"
             $asm.puts "lw $16, 0($sp)"
             $asm.puts "lw $17, 4($sp)"
index a5ab63b..e47f295 100644 (file)
@@ -100,7 +100,7 @@ class RegisterID
             when :quad
                 isX64 ? "%rax" : raise
             else
-                raise
+                raise "Invalid kind #{kind} for name #{name}"
             end
         when "t1", "a1", "r1"
             case kind
@@ -978,9 +978,15 @@ class Instruction
                 $asm.puts "xorpd #{operands[0].x86Operand(:double)}, #{operands[0].x86Operand(:double)}"
             end
         when "pop"
-            $asm.puts "pop #{operands[0].x86Operand(:ptr)}"
+            operands.each {
+                | op |
+                $asm.puts "pop #{op.x86Operand(:ptr)}"
+            }
         when "push"
-            $asm.puts "push #{operands[0].x86Operand(:ptr)}"
+            operands.each {
+                | op |
+                $asm.puts "push #{op.x86Operand(:ptr)}"
+            }
         when "popCalleeSaves"
             if isX64
                 $asm.puts "pop %rbx"