2009-05-09 Maciej Stachowiak <mjs@apple.com>

        Reviewed by Gavin Barraclough.

        Original patch by John McCall. Updated by Cameron Zwarich. Further refined by me.

        - Assorted speedups to property access

        ~.3%-1% speedup on SunSpider

        1) When we know from the structure ID that an object is using inline storage, plant direct
        loads and stores against it; no need to indirect through storage pointer.

        2) Also because of the above, union the property storage pointer with the first inline property
        slot and add an extra inline property slot.

        * assembler/AbstractMacroAssembler.h:
        (JSC::AbstractMacroAssembler::CodeLocationInstruction::CodeLocationInstruction):
        (JSC::AbstractMacroAssembler::CodeLocationInstruction::patchLoadToLEA):
        (JSC::::CodeLocationCommon::instructionAtOffset):
        * assembler/MacroAssembler.h:
        (JSC::MacroAssembler::storePtr):
        * assembler/MacroAssemblerX86.h:
        (JSC::MacroAssemblerX86::store32):
        * assembler/MacroAssemblerX86_64.h:
        (JSC::MacroAssemblerX86_64::storePtr):
        * assembler/X86Assembler.h:
        (JSC::X86Assembler::movq_EAXm):
        (JSC::X86Assembler::movl_rm):
        (JSC::X86Assembler::patchLoadToLEA):
        * jit/JIT.cpp:
        (JSC::JIT::privateCompileMainPass):
        * jit/JIT.h:
        * jit/JITPropertyAccess.cpp:
        (JSC::JIT::compileGetByIdHotPath):
        (JSC::JIT::compilePutByIdHotPath):
        (JSC::JIT::compilePutDirectOffset):
        (JSC::JIT::compileGetDirectOffset):
        (JSC::JIT::privateCompilePutByIdTransition):
        (JSC::JIT::patchGetByIdSelf):
        (JSC::JIT::patchPutByIdReplace):
        (JSC::JIT::privateCompileGetByIdSelf):
        (JSC::JIT::privateCompileGetByIdProto):
        (JSC::JIT::privateCompileGetByIdSelfList):
        (JSC::JIT::privateCompileGetByIdProtoList):
        (JSC::JIT::privateCompileGetByIdChainList):
        (JSC::JIT::privateCompileGetByIdChain):
        (JSC::JIT::privateCompilePutByIdReplace):
        * runtime/JSObject.cpp:
        (JSC::JSObject::mark):
        (JSC::JSObject::removeDirect):
        * runtime/JSObject.h:
        (JSC::JSObject::propertyStorage):
        (JSC::JSObject::getDirect):
        (JSC::JSObject::getOffset):
        (JSC::JSObject::offsetForLocation):
        (JSC::JSObject::locationForOffset):
        (JSC::JSObject::getDirectOffset):
        (JSC::JSObject::putDirectOffset):
        (JSC::JSObject::isUsingInlineStorage):
        (JSC::JSObject::):
        (JSC::JSObject::JSObject):
        (JSC::JSObject::~JSObject):
        (JSC::Structure::isUsingInlineStorage):
        (JSC::JSObject::putDirect):
        (JSC::JSObject::putDirectWithoutTransition):
        (JSC::JSObject::allocatePropertyStorageInline):
        * runtime/Structure.h:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@43432 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/JavaScriptCore/ChangeLog b/JavaScriptCore/ChangeLog
index 7998ab08faeaa3275eb8a19a1f04ee42e23bd9aa..824d530e6ab45dbe4ed028df142cead9c5ad16a1 100644 (file)
--- a/JavaScriptCore/ChangeLog
+++ b/JavaScriptCore/ChangeLog
@@ -1,3 +1,72 @@
+2009-05-09  Maciej Stachowiak  <mjs@apple.com>
+
+        Reviewed by Gavin Barraclough.
+        
+        Original patch by John McCall. Updated by Cameron Zwarich. Further refined by me.
+        
+        - Assorted speedups to property access
+        
+        ~.3%-1% speedup on SunSpider
+        
+        1) When we know from the structure ID that an object is using inline storage, plant direct
+        loads and stores against it; no need to indirect through storage pointer.
+        
+        2) Also because of the above, union the property storage pointer with the first inline property
+        slot and add an extra inline property slot.
+
+        * assembler/AbstractMacroAssembler.h:
+        (JSC::AbstractMacroAssembler::CodeLocationInstruction::CodeLocationInstruction):
+        (JSC::AbstractMacroAssembler::CodeLocationInstruction::patchLoadToLEA):
+        (JSC::::CodeLocationCommon::instructionAtOffset):
+        * assembler/MacroAssembler.h:
+        (JSC::MacroAssembler::storePtr):
+        * assembler/MacroAssemblerX86.h:
+        (JSC::MacroAssemblerX86::store32):
+        * assembler/MacroAssemblerX86_64.h:
+        (JSC::MacroAssemblerX86_64::storePtr):
+        * assembler/X86Assembler.h:
+        (JSC::X86Assembler::movq_EAXm):
+        (JSC::X86Assembler::movl_rm):
+        (JSC::X86Assembler::patchLoadToLEA):
+        * jit/JIT.cpp:
+        (JSC::JIT::privateCompileMainPass):
+        * jit/JIT.h:
+        * jit/JITPropertyAccess.cpp:
+        (JSC::JIT::compileGetByIdHotPath):
+        (JSC::JIT::compilePutByIdHotPath):
+        (JSC::JIT::compilePutDirectOffset):
+        (JSC::JIT::compileGetDirectOffset):
+        (JSC::JIT::privateCompilePutByIdTransition):
+        (JSC::JIT::patchGetByIdSelf):
+        (JSC::JIT::patchPutByIdReplace):
+        (JSC::JIT::privateCompileGetByIdSelf):
+        (JSC::JIT::privateCompileGetByIdProto):
+        (JSC::JIT::privateCompileGetByIdSelfList):
+        (JSC::JIT::privateCompileGetByIdProtoList):
+        (JSC::JIT::privateCompileGetByIdChainList):
+        (JSC::JIT::privateCompileGetByIdChain):
+        (JSC::JIT::privateCompilePutByIdReplace):
+        * runtime/JSObject.cpp:
+        (JSC::JSObject::mark):
+        (JSC::JSObject::removeDirect):
+        * runtime/JSObject.h:
+        (JSC::JSObject::propertyStorage):
+        (JSC::JSObject::getDirect):
+        (JSC::JSObject::getOffset):
+        (JSC::JSObject::offsetForLocation):
+        (JSC::JSObject::locationForOffset):
+        (JSC::JSObject::getDirectOffset):
+        (JSC::JSObject::putDirectOffset):
+        (JSC::JSObject::isUsingInlineStorage):
+        (JSC::JSObject::):
+        (JSC::JSObject::JSObject):
+        (JSC::JSObject::~JSObject):
+        (JSC::Structure::isUsingInlineStorage):
+        (JSC::JSObject::putDirect):
+        (JSC::JSObject::putDirectWithoutTransition):
+        (JSC::JSObject::allocatePropertyStorageInline):
+        * runtime/Structure.h:
+
 2009-05-09  Geoffrey Garen  <ggaren@apple.com>
 
         Reviewed by Gavin Barraclough.

diff --git a/JavaScriptCore/assembler/AbstractMacroAssembler.h b/JavaScriptCore/assembler/AbstractMacroAssembler.h
index 851b6d52e61af6786867e14123eee23e81715932..0033861a211797f440dfd51397659f67efc02a12 100644 (file)
--- a/JavaScriptCore/assembler/AbstractMacroAssembler.h
+++ b/JavaScriptCore/assembler/AbstractMacroAssembler.h
@@ -37,6 +37,7 @@ class AbstractMacroAssembler {
 public:
     class Jump;
     class PatchBuffer;
+    class CodeLocationInstruction;
     class CodeLocationLabel;
     class CodeLocationJump;
     class CodeLocationCall;
@@ -406,6 +407,7 @@ public:
         // and the labels will always be a fixed distance apart, these
         // methods may be used to recover a handle that has nopw been
         // retained, based on a known fixed relative offset from one that has.
+        CodeLocationInstruction instructionAtOffset(int offset);
         CodeLocationLabel labelAtOffset(int offset);
         CodeLocationJump jumpAtOffset(int offset);
         CodeLocationCall callAtOffset(int offset);
@@ -424,6 +426,27 @@ public:
         void* m_location;
     };
 
+    // CodeLocationInstruction:
+    //
+    // An arbitrary instruction in the JIT code.
+    class CodeLocationInstruction : public CodeLocationCommon {
+        friend class CodeLocationCommon;
+    public:
+        CodeLocationInstruction()
+        {
+        }
+
+        void patchLoadToLEA() {
+            AssemblerType::patchLoadToLEA(reinterpret_cast<intptr_t>(this->m_location));
+        }
+
+    private:
+        explicit CodeLocationInstruction(void* location)
+            : CodeLocationCommon(location)
+        {
+        }
+    };
+
     // CodeLocationLabel:
     //
     // A point in the JIT code maked with a label.
@@ -803,6 +826,12 @@ protected:
 };
 
 
+template <class AssemblerType>
+typename AbstractMacroAssembler<AssemblerType>::CodeLocationInstruction AbstractMacroAssembler<AssemblerType>::CodeLocationCommon::instructionAtOffset(int offset)
+{
+    return typename AbstractMacroAssembler::CodeLocationInstruction(reinterpret_cast<char*>(m_location) + offset);
+}
+
 template <class AssemblerType>
 typename AbstractMacroAssembler<AssemblerType>::CodeLocationLabel AbstractMacroAssembler<AssemblerType>::CodeLocationCommon::labelAtOffset(int offset)
 {

diff --git a/JavaScriptCore/assembler/MacroAssembler.h b/JavaScriptCore/assembler/MacroAssembler.h
index 71ac1f64fa6312c764ce1054ef03461f2f0fd50d..f341267e75dc722ade84e09df3220e817ad2ddf1 100644 (file)
--- a/JavaScriptCore/assembler/MacroAssembler.h
+++ b/JavaScriptCore/assembler/MacroAssembler.h
@@ -242,6 +242,11 @@ public:
         store32(src, address);
     }
 
+    void storePtr(RegisterID src, void* address)
+    {
+        store32(src, address);
+    }
+
     void storePtr(ImmPtr imm, ImplicitAddress address)
     {
         store32(Imm32(imm), address);

diff --git a/JavaScriptCore/assembler/MacroAssemblerX86.h b/JavaScriptCore/assembler/MacroAssemblerX86.h
index b85b8b232f4c8b19f45b440ca96b74f2be20890a..89ecd7f24b03751c9d4081806dcb69b19d2f5132 100644 (file)
--- a/JavaScriptCore/assembler/MacroAssemblerX86.h
+++ b/JavaScriptCore/assembler/MacroAssemblerX86.h
@@ -70,6 +70,11 @@ public:
         m_assembler.movl_i32m(imm.m_value, address);
     }
 
+    void store32(RegisterID src, void* address)
+    {
+        m_assembler.movl_rm(src, address);
+    }
+
     Jump branch32(Condition cond, AbsoluteAddress left, RegisterID right)
     {
         m_assembler.cmpl_rm(right, left.m_ptr);

diff --git a/JavaScriptCore/assembler/MacroAssemblerX86_64.h b/JavaScriptCore/assembler/MacroAssemblerX86_64.h
index a06e66fc2f067024ab190b9cb026ec763eb0179a..753178be51702ac23be06f4ea44e936854c66cea 100644 (file)
--- a/JavaScriptCore/assembler/MacroAssemblerX86_64.h
+++ b/JavaScriptCore/assembler/MacroAssemblerX86_64.h
@@ -241,6 +241,17 @@ public:
     {
         m_assembler.movq_rm(src, address.offset, address.base, address.index, address.scale);
     }
+    
+    void storePtr(RegisterID src, void* address)
+    {
+        if (src == X86::eax)
+            m_assembler.movq_EAXm(address);
+        else {
+            swap(X86::eax, src);
+            m_assembler.movq_EAXm(address);
+            swap(X86::eax, src);
+        }
+    }
 
     void storePtr(ImmPtr imm, ImplicitAddress address)
     {

diff --git a/JavaScriptCore/assembler/X86Assembler.h b/JavaScriptCore/assembler/X86Assembler.h
index ac7c40aa0ad1aaf265679b8c8d7ad42c1303c848..81653a8646b65bc94e14a1436fdc43c986edc181 100644 (file)
--- a/JavaScriptCore/assembler/X86Assembler.h
+++ b/JavaScriptCore/assembler/X86Assembler.h
@@ -937,6 +937,12 @@ public:
         m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
     }
 
+    void movq_EAXm(void* addr)
+    {
+        m_formatter.oneByteOp64(OP_MOV_OvEAX);
+        m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+    }
+
     void movq_mr(int offset, RegisterID base, RegisterID dst)
     {
         m_formatter.oneByteOp64(OP_MOV_GvEv, dst, base, offset);
@@ -971,6 +977,14 @@ public:
     
     
 #else
+    void movl_rm(RegisterID src, void* addr)
+    {
+        if (src == X86::eax)
+            movl_EAXm(addr);
+        else 
+            m_formatter.oneByteOp(OP_MOV_EvGv, src, addr);
+    }
+    
     void movl_mr(void* addr, RegisterID dst)
     {
         if (dst == X86::eax)
@@ -1287,6 +1301,12 @@ public:
         ASSERT(linkOffset == static_cast<int>(linkOffset));
         reinterpret_cast<int*>(reinterpret_cast<ptrdiff_t>(code) + from.m_offset)[-1] = linkOffset;
     }
+
+    static void patchLoadToLEA(intptr_t where)
+    {
+        char* ptr = reinterpret_cast<char*>(where);
+        ptr[0] = OP_LEA;
+    }
     
     static void patchJump(intptr_t where, void* destination)
     {

diff --git a/JavaScriptCore/jit/JIT.cpp b/JavaScriptCore/jit/JIT.cpp
index b6e57d008d950b5dcbe81875b3e5d7c533c4ab59..b1b3c4f12865d335f9c1c47b6bca3f68201deefc 100644 (file)
--- a/JavaScriptCore/jit/JIT.cpp
+++ b/JavaScriptCore/jit/JIT.cpp
@@ -594,7 +594,8 @@ void JIT::privateCompileMainPass()
             Jump noMatch = branchPtr(NotEqual, regT1, Address(regT0, FIELD_OFFSET(JSCell, m_structure))); // Structures don't match
 
             // Load cached property
-            loadPtr(Address(regT0, FIELD_OFFSET(JSGlobalObject, m_propertyStorage)), regT0);
+            // Assume that the global object always uses external storage.
+            loadPtr(Address(regT0, FIELD_OFFSET(JSGlobalObject, m_externalStorage)), regT0);
             load32(offsetAddr, regT1);
             loadPtr(BaseIndex(regT0, regT1, ScalePtr), regT0);
             emitPutVirtualRegister(currentInstruction[1].u.operand);

diff --git a/JavaScriptCore/jit/JIT.h b/JavaScriptCore/jit/JIT.h
index b25d150d525a2908877a3307b95a87ebd84adaec..3bd33c0f4d9f99d67aba84a0f711f7e1b6b58e29 100644 (file)
--- a/JavaScriptCore/jit/JIT.h
+++ b/JavaScriptCore/jit/JIT.h
@@ -226,10 +226,14 @@ namespace JSC {
 #if PLATFORM(X86_64)
         // These architecture specific value are used to enable patching - see comment on op_put_by_id.
         static const int patchOffsetPutByIdStructure = 10;
+        static const int patchOffsetPutByIdExternalLoad = 20;
+        static const int patchLengthPutByIdExternalLoad = 4;
         static const int patchOffsetPutByIdPropertyMapOffset = 31;
         // These architecture specific value are used to enable patching - see comment on op_get_by_id.
         static const int patchOffsetGetByIdStructure = 10;
         static const int patchOffsetGetByIdBranchToSlowCase = 20;
+        static const int patchOffsetGetByIdExternalLoad = 20;
+        static const int patchLengthGetByIdExternalLoad = 4;
         static const int patchOffsetGetByIdPropertyMapOffset = 31;
         static const int patchOffsetGetByIdPutResult = 31;
 #if ENABLE(OPCODE_SAMPLING)
@@ -241,10 +245,14 @@ namespace JSC {
 #else
         // These architecture specific value are used to enable patching - see comment on op_put_by_id.
         static const int patchOffsetPutByIdStructure = 7;
+        static const int patchOffsetPutByIdExternalLoad = 13;
+        static const int patchLengthPutByIdExternalLoad = 3;
         static const int patchOffsetPutByIdPropertyMapOffset = 22;
         // These architecture specific value are used to enable patching - see comment on op_get_by_id.
         static const int patchOffsetGetByIdStructure = 7;
         static const int patchOffsetGetByIdBranchToSlowCase = 13;
+        static const int patchOffsetGetByIdExternalLoad = 13;
+        static const int patchLengthGetByIdExternalLoad = 3;
         static const int patchOffsetGetByIdPropertyMapOffset = 22;
         static const int patchOffsetGetByIdPutResult = 22;
 #if ENABLE(OPCODE_SAMPLING)
@@ -380,6 +388,10 @@ namespace JSC {
         enum CompileOpStrictEqType { OpStrictEq, OpNStrictEq };
         void compileOpStrictEq(Instruction* instruction, CompileOpStrictEqType type);
 
+        void compileGetDirectOffset(RegisterID base, RegisterID result, Structure* structure, size_t cachedOffset);
+        void compileGetDirectOffset(JSObject* base, RegisterID temp, RegisterID result, size_t cachedOffset);
+        void compilePutDirectOffset(RegisterID base, RegisterID value, Structure* structure, size_t cachedOffset);
+
         void compileFastArith_op_add(Instruction*);
         void compileFastArith_op_sub(Instruction*);
         void compileFastArith_op_mul(Instruction*);

diff --git a/JavaScriptCore/jit/JITPropertyAccess.cpp b/JavaScriptCore/jit/JITPropertyAccess.cpp
index cdd7d87cc3d0e73c6db5ef16925f6da0a712c140..6aa758d2fa8ab4ae4e49bb256e450d1a0b5161a1 100644 (file)
--- a/JavaScriptCore/jit/JITPropertyAccess.cpp
+++ b/JavaScriptCore/jit/JITPropertyAccess.cpp
@@ -108,7 +108,12 @@ void JIT::compileGetByIdHotPath(int resultVReg, int baseVReg, Identifier*, unsig
     ASSERT(differenceBetween(hotPathBegin, structureToCompare) == patchOffsetGetByIdStructure);
     ASSERT(differenceBetween(hotPathBegin, structureCheck) == patchOffsetGetByIdBranchToSlowCase);
 
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
+    Label externalLoad(this);
+    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_externalStorage)), regT0);
+    Label externalLoadComplete(this);
+    ASSERT(differenceBetween(hotPathBegin, externalLoad) == patchOffsetGetByIdExternalLoad);
+    ASSERT(differenceBetween(externalLoad, externalLoadComplete) == patchLengthGetByIdExternalLoad);
+
     DataLabel32 displacementLabel = loadPtrWithAddressOffsetPatch(Address(regT0, patchGetByIdDefaultOffset), regT0);
     ASSERT(differenceBetween(hotPathBegin, displacementLabel) == patchOffsetGetByIdPropertyMapOffset);
 
@@ -163,7 +168,12 @@ void JIT::compilePutByIdHotPath(int baseVReg, Identifier*, int valueVReg, unsign
     ASSERT(differenceBetween(hotPathBegin, structureToCompare) == patchOffsetPutByIdStructure);
 
     // Plant a load from a bogus ofset in the object's property map; we will patch this later, if it is to be used.
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
+    Label externalLoad(this);
+    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_externalStorage)), regT0);
+    Label externalLoadComplete(this);
+    ASSERT(differenceBetween(hotPathBegin, externalLoad) == patchOffsetPutByIdExternalLoad);
+    ASSERT(differenceBetween(externalLoad, externalLoadComplete) == patchLengthPutByIdExternalLoad);
+
     DataLabel32 displacementLabel = storePtrWithAddressOffsetPatch(regT1, Address(regT0, patchGetByIdDefaultOffset));
     ASSERT(differenceBetween(hotPathBegin, displacementLabel) == patchOffsetPutByIdPropertyMapOffset);
 }
@@ -183,6 +193,40 @@ void JIT::compilePutByIdSlowCase(int baseVReg, Identifier* ident, int, Vector<Sl
     m_propertyAccessCompilationInfo[propertyAccessInstructionIndex].callReturnLocation = call;
 }
 
+// Compile a store into an object's property storage.  May overwrite the
+// value in objectReg.
+void JIT::compilePutDirectOffset(RegisterID base, RegisterID value, Structure* structure, size_t cachedOffset)
+{
+    int offset = cachedOffset * sizeof(JSValue);
+    if (structure->isUsingInlineStorage())
+        offset += FIELD_OFFSET(JSObject, m_inlineStorage);
+    else
+        loadPtr(Address(base, FIELD_OFFSET(JSObject, m_externalStorage)), base);
+    storePtr(value, Address(base, offset));
+}
+
+// Compile a load from an object's property storage.  May overwrite base.
+void JIT::compileGetDirectOffset(RegisterID base, RegisterID result, Structure* structure, size_t cachedOffset)
+{
+    int offset = cachedOffset * sizeof(JSValue);
+    if (structure->isUsingInlineStorage())
+        offset += FIELD_OFFSET(JSObject, m_inlineStorage);
+    else
+        loadPtr(Address(base, FIELD_OFFSET(JSObject, m_externalStorage)), base);
+    loadPtr(Address(base, offset), result);
+}
+
+void JIT::compileGetDirectOffset(JSObject* base, RegisterID temp, RegisterID result, size_t cachedOffset)
+{
+    if (base->isUsingInlineStorage())
+        loadPtr(static_cast<void*>(&base->m_inlineStorage[cachedOffset]), result);
+    else {
+        PropertyStorage* protoPropertyStorage = &base->m_externalStorage;
+        loadPtr(static_cast<void*>(protoPropertyStorage), temp);
+        loadPtr(Address(temp, cachedOffset * sizeof(JSValue)), result);
+    } 
+}
+
 static JSObject* resizePropertyStorage(JSObject* baseObject, int32_t oldSize, int32_t newSize)
 {
     baseObject->allocatePropertyStorage(oldSize, newSize);
@@ -253,8 +297,7 @@ void JIT::privateCompilePutByIdTransition(StructureStubInfo* stubInfo, Structure
     storePtr(ImmPtr(newStructure), Address(regT0, FIELD_OFFSET(JSCell, m_structure)));
 
     // write the value
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
-    storePtr(regT1, Address(regT0, cachedOffset * sizeof(JSValue)));
+    compilePutDirectOffset(regT0, regT1, newStructure, cachedOffset);
 
     ret();
     
@@ -282,9 +325,16 @@ void JIT::patchGetByIdSelf(StructureStubInfo* stubInfo, Structure* structure, si
     // Should probably go to JITStubs::cti_op_get_by_id_fail, but that doesn't do anything interesting right now.
     returnAddress.relinkCallerToFunction(JITStubs::cti_op_get_by_id_self_fail);
 
+    int offset = sizeof(JSValue) * cachedOffset;
+
+    // If we're patching to use inline storage, convert the initial load to a lea; this avoids the extra load
+    // and makes the subsequent load's offset automatically correct
+    if (structure->isUsingInlineStorage())
+        stubInfo->hotPathBegin.instructionAtOffset(patchOffsetGetByIdExternalLoad).patchLoadToLEA();
+
     // Patch the offset into the propoerty map to load from, then patch the Structure to look for.
     stubInfo->hotPathBegin.dataLabelPtrAtOffset(patchOffsetGetByIdStructure).repatch(structure);
-    stubInfo->hotPathBegin.dataLabel32AtOffset(patchOffsetGetByIdPropertyMapOffset).repatch(cachedOffset * sizeof(JSValue));
+    stubInfo->hotPathBegin.dataLabel32AtOffset(patchOffsetGetByIdPropertyMapOffset).repatch(offset);
 }
 
 void JIT::patchPutByIdReplace(StructureStubInfo* stubInfo, Structure* structure, size_t cachedOffset, ProcessorReturnAddress returnAddress)
@@ -293,9 +343,16 @@ void JIT::patchPutByIdReplace(StructureStubInfo* stubInfo, Structure* structure,
     // Should probably go to JITStubs::cti_op_put_by_id_fail, but that doesn't do anything interesting right now.
     returnAddress.relinkCallerToFunction(JITStubs::cti_op_put_by_id_generic);
 
+    int offset = sizeof(JSValue) * cachedOffset;
+
+    // If we're patching to use inline storage, convert the initial load to a lea; this avoids the extra load
+    // and makes the subsequent load's offset automatically correct
+    if (structure->isUsingInlineStorage())
+        stubInfo->hotPathBegin.instructionAtOffset(patchOffsetGetByIdExternalLoad).patchLoadToLEA();
+
     // Patch the offset into the propoerty map to load from, then patch the Structure to look for.
     stubInfo->hotPathBegin.dataLabelPtrAtOffset(patchOffsetPutByIdStructure).repatch(structure);
-    stubInfo->hotPathBegin.dataLabel32AtOffset(patchOffsetPutByIdPropertyMapOffset).repatch(cachedOffset * sizeof(JSValue));
+    stubInfo->hotPathBegin.dataLabel32AtOffset(patchOffsetPutByIdPropertyMapOffset).repatch(offset);
 }
 
 void JIT::privateCompilePatchGetArrayLength(ProcessorReturnAddress returnAddress)
@@ -344,8 +401,7 @@ void JIT::privateCompileGetByIdSelf(StructureStubInfo* stubInfo, Structure* stru
     Jump failureCases2 = checkStructure(regT0, structure);
 
     // Checks out okay! - getDirectOffset
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
-    loadPtr(Address(regT0, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(regT0, regT0, structure, cachedOffset);
     ret();
 
     Call failureCases1Call = makeTailRecursiveCall(failureCases1);
@@ -385,9 +441,7 @@ void JIT::privateCompileGetByIdProto(StructureStubInfo* stubInfo, Structure* str
 #endif
 
     // Checks out okay! - getDirectOffset
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(static_cast<void*>(protoPropertyStorage), regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
 
     Jump success = jump();
 
@@ -423,9 +477,7 @@ void JIT::privateCompileGetByIdProto(StructureStubInfo* stubInfo, Structure* str
     Jump failureCases3 = branchPtr(NotEqual, AbsoluteAddress(prototypeStructureAddress), ImmPtr(prototypeStructure));
 
     // Checks out okay! - getDirectOffset
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(protoPropertyStorage, regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
 
     ret();
 
@@ -446,8 +498,7 @@ void JIT::privateCompileGetByIdProto(StructureStubInfo* stubInfo, Structure* str
 void JIT::privateCompileGetByIdSelfList(StructureStubInfo* stubInfo, PolymorphicAccessStructureList* polymorphicStructures, int currentIndex, Structure* structure, size_t cachedOffset)
 {
     Jump failureCase = checkStructure(regT0, structure);
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
-    loadPtr(Address(regT0, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(regT0, regT0, structure, cachedOffset);
     Jump success = jump();
 
     void* code = m_assembler.executableCopy(m_codeBlock->executablePool());
@@ -493,9 +544,7 @@ void JIT::privateCompileGetByIdProtoList(StructureStubInfo* stubInfo, Polymorphi
 #endif
 
     // Checks out okay! - getDirectOffset
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(protoPropertyStorage, regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
 
     Jump success = jump();
 
@@ -549,9 +598,7 @@ void JIT::privateCompileGetByIdChainList(StructureStubInfo* stubInfo, Polymorphi
     }
     ASSERT(protoObject);
 
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(protoPropertyStorage, regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
     Jump success = jump();
 
     void* code = m_assembler.executableCopy(m_codeBlock->executablePool());
@@ -609,9 +656,7 @@ void JIT::privateCompileGetByIdChain(StructureStubInfo* stubInfo, Structure* str
     }
     ASSERT(protoObject);
 
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(protoPropertyStorage, regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
     Jump success = jump();
 
     void* code = m_assembler.executableCopy(m_codeBlock->executablePool());
@@ -657,9 +702,8 @@ void JIT::privateCompileGetByIdChain(StructureStubInfo* stubInfo, Structure* str
     }
     ASSERT(protoObject);
 
-    PropertyStorage* protoPropertyStorage = &protoObject->m_propertyStorage;
-    loadPtr(protoPropertyStorage, regT1);
-    loadPtr(Address(regT1, cachedOffset * sizeof(JSValue)), regT0);
+    compileGetDirectOffset(protoObject, regT1, regT0, cachedOffset);
+    compilePutDirectOffset(regT0, regT1, structure, cachedOffset);
     ret();
 
     void* code = m_assembler.executableCopy(m_codeBlock->executablePool());
@@ -679,8 +723,7 @@ void JIT::privateCompilePutByIdReplace(StructureStubInfo* stubInfo, Structure* s
     Jump failureCases2 = checkStructure(regT0, structure);
 
     // checks out okay! - putDirectOffset
-    loadPtr(Address(regT0, FIELD_OFFSET(JSObject, m_propertyStorage)), regT0);
-    storePtr(regT1, Address(regT0, cachedOffset * sizeof(JSValue)));
+    compilePutDirectOffset(regT0, regT1, structure, cachedOffset);
     ret();
 
     Call failureCases1Call = makeTailRecursiveCall(failureCases1);

diff --git a/JavaScriptCore/runtime/JSObject.cpp b/JavaScriptCore/runtime/JSObject.cpp
index 905292e59a8abf35aabb0489a23f1bedfac4d8aa..15fc09c0cd2090d6dc67ca8af1bc65e55b32b516 100644 (file)
--- a/JavaScriptCore/runtime/JSObject.cpp
+++ b/JavaScriptCore/runtime/JSObject.cpp
@@ -69,9 +69,11 @@ void JSObject::mark()
     JSCell::mark();
     m_structure->mark();
 
+    PropertyStorage storage = propertyStorage();
+
     size_t storageSize = m_structure->propertyStorageSize();
     for (size_t i = 0; i < storageSize; ++i) {
-        JSValue v = m_propertyStorage[i];
+        JSValue v = JSValue::decode(storage[i]);
         if (!v.marked())
             v.mark();
     }
@@ -471,14 +473,14 @@ void JSObject::removeDirect(const Identifier& propertyName)
     if (m_structure->isDictionary()) {
         offset = m_structure->removePropertyWithoutTransition(propertyName);
         if (offset != WTF::notFound)
-            m_propertyStorage[offset] = jsUndefined();
+            putDirectOffset(offset, jsUndefined());
         return;
     }
 
     RefPtr<Structure> structure = Structure::removePropertyTransition(m_structure, propertyName, offset);
-    if (offset != WTF::notFound)
-        m_propertyStorage[offset] = jsUndefined();
     setStructure(structure.release());
+    if (offset != WTF::notFound)
+        putDirectOffset(offset, jsUndefined());
 }
 
 void JSObject::putDirectFunction(ExecState* exec, InternalFunction* function, unsigned attr)

diff --git a/JavaScriptCore/runtime/JSObject.h b/JavaScriptCore/runtime/JSObject.h
index d0b6f059aec1856f68108a91f92e85179a823fdf..8ec3eb0f26d303376bb39094c40091825bde81c7 100644 (file)
--- a/JavaScriptCore/runtime/JSObject.h
+++ b/JavaScriptCore/runtime/JSObject.h
@@ -51,7 +51,8 @@ namespace JSC {
         Function     = 1 << 4,  // property is a function - only used by static hashtables
     };
 
-    typedef JSValue* PropertyStorage;
+    typedef EncodedJSValue* PropertyStorage;
+    typedef EncodedJSValue const * ConstPropertyStorage;
 
     class JSObject : public JSCell {
         friend class BatchedTransitionOptimizer;
@@ -75,7 +76,8 @@ namespace JSC {
         void setStructure(PassRefPtr<Structure>);
         Structure* inheritorID();
 
-        PropertyStorage& propertyStorage() { return m_propertyStorage; }
+        ConstPropertyStorage propertyStorage() const { return (isUsingInlineStorage() ? m_inlineStorage : m_externalStorage); }
+        PropertyStorage propertyStorage() { return (isUsingInlineStorage() ? m_inlineStorage : m_externalStorage); }
 
         virtual UString className() const;
 
@@ -125,7 +127,12 @@ namespace JSC {
         JSValue getDirect(const Identifier& propertyName) const
         {
             size_t offset = m_structure->get(propertyName);
-            return offset != WTF::notFound ? m_propertyStorage[offset] : JSValue();
+            return offset != WTF::notFound ? getDirectOffset(offset) : JSValue();
+        }
+
+        size_t getOffset(const Identifier& propertyName)
+        {
+            return m_structure->get(propertyName);
         }
 
         JSValue* getDirectLocation(const Identifier& propertyName)
@@ -140,14 +147,19 @@ namespace JSC {
             return offset != WTF::notFound ? locationForOffset(offset) : 0;
         }
 
-        size_t offsetForLocation(JSValue* location)
+        size_t offsetForLocation(JSValue* location) const
+        {
+            return location - reinterpret_cast<JSValue const *>(propertyStorage());
+        }
+
+        JSValue const * locationForOffset(size_t offset) const
         {
-            return location - m_propertyStorage;
+            return reinterpret_cast<JSValue const *>(&propertyStorage()[offset]);
         }
 
         JSValue* locationForOffset(size_t offset)
         {
-            return &m_propertyStorage[offset];
+            return reinterpret_cast<JSValue*>(&propertyStorage()[offset]);
         }
 
         void transitionTo(Structure*);
@@ -163,8 +175,8 @@ namespace JSC {
         void putDirectFunctionWithoutTransition(ExecState* exec, InternalFunction* function, unsigned attr = 0);
 
         // Fast access to known property offsets.
-        JSValue getDirectOffset(size_t offset) { return m_propertyStorage[offset]; }
-        void putDirectOffset(size_t offset, JSValue value) { m_propertyStorage[offset] = value; }
+        JSValue getDirectOffset(size_t offset) const { return JSValue::decode(propertyStorage()[offset]); }
+        void putDirectOffset(size_t offset, JSValue value) { propertyStorage()[offset] = JSValue::encode(value); }
 
         void fillGetterPropertySlot(PropertySlot&, JSValue* location);
 
@@ -181,9 +193,9 @@ namespace JSC {
 
         void allocatePropertyStorage(size_t oldSize, size_t newSize);
         void allocatePropertyStorageInline(size_t oldSize, size_t newSize);
-        bool usingInlineStorage() const { return m_propertyStorage == m_inlineStorage; }
+        bool isUsingInlineStorage() const { return m_structure->isUsingInlineStorage(); }
 
-        static const size_t inlineStorageCapacity = 2;
+        static const size_t inlineStorageCapacity = 3;
         static const size_t nonInlineBaseStorageCapacity = 16;
 
         static PassRefPtr<Structure> createStructure(JSValue prototype)
@@ -202,8 +214,10 @@ namespace JSC {
 
         RefPtr<Structure> m_inheritorID;
 
-        PropertyStorage m_propertyStorage;        
-        JSValue m_inlineStorage[inlineStorageCapacity];
+        union {
+            PropertyStorage m_externalStorage;
+            EncodedJSValue m_inlineStorage[inlineStorageCapacity];
+        };
     };
 
     JSObject* asObject(JSValue);
@@ -218,7 +232,6 @@ inline JSObject* asObject(JSValue value)
 
 inline JSObject::JSObject(PassRefPtr<Structure> structure)
     : JSCell(structure.releaseRef()) // ~JSObject balances this ref()
-    , m_propertyStorage(m_inlineStorage)
 {
     ASSERT(m_structure);
     ASSERT(m_structure->propertyStorageCapacity() == inlineStorageCapacity);
@@ -229,8 +242,8 @@ inline JSObject::JSObject(PassRefPtr<Structure> structure)
 inline JSObject::~JSObject()
 {
     ASSERT(m_structure);
-    if (m_propertyStorage != m_inlineStorage)
-        delete [] m_propertyStorage;
+    if (!isUsingInlineStorage())
+        delete [] m_externalStorage;
     m_structure->deref();
 }
 
@@ -259,6 +272,11 @@ inline Structure* JSObject::inheritorID()
     return createInheritorID();
 }
 
+inline bool Structure::isUsingInlineStorage() const
+{
+    return (propertyStorageCapacity() == JSObject::inlineStorageCapacity);
+}
+
 inline bool JSCell::isObject(const ClassInfo* info) const
 {
     for (const ClassInfo* ci = classInfo(); ci; ci = ci->parentClass) {
@@ -394,7 +412,7 @@ inline void JSObject::putDirect(const Identifier& propertyName, JSValue value, u
         if (offset != WTF::notFound) {
             if (checkReadOnly && currentAttributes & ReadOnly)
                 return;
-            m_propertyStorage[offset] = value;
+            putDirectOffset(offset, value);
             slot.setExistingProperty(this, offset);
             return;
         }
@@ -405,7 +423,7 @@ inline void JSObject::putDirect(const Identifier& propertyName, JSValue value, u
             allocatePropertyStorage(currentCapacity, m_structure->propertyStorageCapacity());
 
         ASSERT(offset < m_structure->propertyStorageCapacity());
-        m_propertyStorage[offset] = value;
+        putDirectOffset(offset, value);
         slot.setNewProperty(this, offset);
         return;
     }
@@ -417,10 +435,10 @@ inline void JSObject::putDirect(const Identifier& propertyName, JSValue value, u
             allocatePropertyStorage(currentCapacity, structure->propertyStorageCapacity());
 
         ASSERT(offset < structure->propertyStorageCapacity());
-        m_propertyStorage[offset] = value;
+        setStructure(structure.release());
+        putDirectOffset(offset, value);
         slot.setNewProperty(this, offset);
         slot.setWasTransition(true);
-        setStructure(structure.release());
         return;
     }
 
@@ -429,7 +447,7 @@ inline void JSObject::putDirect(const Identifier& propertyName, JSValue value, u
     if (offset != WTF::notFound) {
         if (checkReadOnly && currentAttributes & ReadOnly)
             return;
-        m_propertyStorage[offset] = value;
+        putDirectOffset(offset, value);
         slot.setExistingProperty(this, offset);
         return;
     }
@@ -439,10 +457,10 @@ inline void JSObject::putDirect(const Identifier& propertyName, JSValue value, u
         allocatePropertyStorage(currentCapacity, structure->propertyStorageCapacity());
 
     ASSERT(offset < structure->propertyStorageCapacity());
-    m_propertyStorage[offset] = value;
+    setStructure(structure.release());
+    putDirectOffset(offset, value);
     slot.setNewProperty(this, offset);
     slot.setWasTransition(true);
-    setStructure(structure.release());
 }
 
 inline void JSObject::putDirectWithoutTransition(const Identifier& propertyName, JSValue value, unsigned attributes)
@@ -451,7 +469,7 @@ inline void JSObject::putDirectWithoutTransition(const Identifier& propertyName,
     size_t offset = m_structure->addPropertyWithoutTransition(propertyName, attributes);
     if (currentCapacity != m_structure->propertyStorageCapacity())
         allocatePropertyStorage(currentCapacity, m_structure->propertyStorageCapacity());
-    m_propertyStorage[offset] = value;
+    putDirectOffset(offset, value);
 }
 
 inline void JSObject::transitionTo(Structure* newStructure)
@@ -540,14 +558,20 @@ ALWAYS_INLINE void JSObject::allocatePropertyStorageInline(size_t oldSize, size_
 {
     ASSERT(newSize > oldSize);
 
-    JSValue* oldPropertyStorage = m_propertyStorage;
-    m_propertyStorage = new JSValue[newSize];
+    // It's important that this function not rely on m_structure, since
+    // we might be in the middle of a transition.
+    bool wasInline = (oldSize == JSObject::inlineStorageCapacity);
+
+    PropertyStorage oldPropertyStorage = (wasInline ? m_inlineStorage : m_externalStorage);
+    PropertyStorage newPropertyStorage = new EncodedJSValue[newSize];
 
     for (unsigned i = 0; i < oldSize; ++i)
-        m_propertyStorage[i] = oldPropertyStorage[i];
+       newPropertyStorage[i] = oldPropertyStorage[i];
 
-    if (oldPropertyStorage != m_inlineStorage)
+    if (!wasInline)
         delete [] oldPropertyStorage;
+
+    m_externalStorage = newPropertyStorage;
 }
 
 } // namespace JSC

diff --git a/JavaScriptCore/runtime/Structure.h b/JavaScriptCore/runtime/Structure.h
index 946b6c70e597fde5b6b9f6716ce442b0cbfdc284..6d869b3a0c02c2b09b6a4ea37f083ea67d96a72f 100644 (file)
--- a/JavaScriptCore/runtime/Structure.h
+++ b/JavaScriptCore/runtime/Structure.h
@@ -95,6 +95,7 @@ namespace JSC {
         void growPropertyStorageCapacity();
         size_t propertyStorageCapacity() const { return m_propertyStorageCapacity; }
         size_t propertyStorageSize() const { return m_propertyTable ? m_propertyTable->keyCount + (m_propertyTable->deletedOffsets ? m_propertyTable->deletedOffsets->size() : 0) : m_offset + 1; }
+        bool isUsingInlineStorage() const;
 
         size_t get(const Identifier& propertyName);
         size_t get(const Identifier& propertyName, unsigned& attributes);