Implementing caching transition puts that need to reallocate with indexing storage
authorfpizlo@apple.com <fpizlo@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 8 Apr 2016 02:11:48 +0000 (02:11 +0000)
committerfpizlo@apple.com <fpizlo@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 8 Apr 2016 02:11:48 +0000 (02:11 +0000)
https://bugs.webkit.org/show_bug.cgi?id=130914

Reviewed by Saam Barati.

Source/JavaScriptCore:

This enables the IC's put_by_id path to handle reallocating the out-of-line storage even if
the butterfly has indexing storage. Like the DFG, we do this by calling operations that
reallocate the butterfly. Those use JSObject API and do all of the nasty work for us, like
triggering a barrier.

This does a bunch of refactoring to how PolymorphicAccess makes calls. It's a lot easier to
do it now because the hard work is hidden under AccessGenerationState methods. This means
that custom accessors now share logic with put_by_id transitions.

* bytecode/PolymorphicAccess.cpp:
(JSC::AccessGenerationState::succeed):
(JSC::AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling):
(JSC::AccessGenerationState::preserveLiveRegistersToStackForCall):
(JSC::AccessGenerationState::originalCallSiteIndex):
(JSC::AccessGenerationState::emitExplicitExceptionHandler):
(JSC::AccessCase::AccessCase):
(JSC::AccessCase::transition):
(JSC::AccessCase::generate):
(JSC::PolymorphicAccess::regenerate):
* bytecode/PolymorphicAccess.h:
(JSC::AccessGenerationState::needsToRestoreRegistersIfException):
(JSC::AccessGenerationState::liveRegistersToPreserveAtExceptionHandlingCallSite):
* dfg/DFGOperations.cpp:
* dfg/DFGOperations.h:
* jit/JITOperations.cpp:
* jit/JITOperations.h:

LayoutTests:

* js/regress/put-by-id-transition-with-indexing-header-expected.txt: Added.
* js/regress/put-by-id-transition-with-indexing-header.html: Added.
* js/regress/script-tests/put-by-id-transition-with-indexing-header.js: Added.
(allocate):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@199209 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/ChangeLog
LayoutTests/js/regress/put-by-id-transition-with-indexing-header-expected.txt [new file with mode: 0644]
LayoutTests/js/regress/put-by-id-transition-with-indexing-header.html [new file with mode: 0644]
LayoutTests/js/regress/script-tests/put-by-id-transition-with-indexing-header.js [new file with mode: 0644]
Source/JavaScriptCore/ChangeLog
Source/JavaScriptCore/bytecode/PolymorphicAccess.cpp
Source/JavaScriptCore/bytecode/PolymorphicAccess.h
Source/JavaScriptCore/dfg/DFGOperations.cpp
Source/JavaScriptCore/dfg/DFGOperations.h
Source/JavaScriptCore/jit/JITOperations.cpp
Source/JavaScriptCore/jit/JITOperations.h

index 6c8afad..aa1169c 100644 (file)
@@ -1,3 +1,15 @@
+2016-04-07  Filip Pizlo  <fpizlo@apple.com>
+
+        Implementing caching transition puts that need to reallocate with indexing storage
+        https://bugs.webkit.org/show_bug.cgi?id=130914
+
+        Reviewed by Saam Barati.
+
+        * js/regress/put-by-id-transition-with-indexing-header-expected.txt: Added.
+        * js/regress/put-by-id-transition-with-indexing-header.html: Added.
+        * js/regress/script-tests/put-by-id-transition-with-indexing-header.js: Added.
+        (allocate):
+
 2016-04-07  Ada Chan  <adachan@apple.com>
 
         Roll out the css change in mediaControlsApple.css that has been causing assertions in layout for multiple tests
diff --git a/LayoutTests/js/regress/put-by-id-transition-with-indexing-header-expected.txt b/LayoutTests/js/regress/put-by-id-transition-with-indexing-header-expected.txt
new file mode 100644 (file)
index 0000000..82adbeb
--- /dev/null
@@ -0,0 +1,10 @@
+JSRegress/put-by-id-transition-with-indexing-header
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS no exception thrown
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
diff --git a/LayoutTests/js/regress/put-by-id-transition-with-indexing-header.html b/LayoutTests/js/regress/put-by-id-transition-with-indexing-header.html
new file mode 100644 (file)
index 0000000..4b20500
--- /dev/null
@@ -0,0 +1,12 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html>
+<head>
+<script src="../../resources/js-test-pre.js"></script>
+</head>
+<body>
+<script src="../../resources/regress-pre.js"></script>
+<script src="script-tests/put-by-id-transition-with-indexing-header.js"></script>
+<script src="../../resources/regress-post.js"></script>
+<script src="../../resources/js-test-post.js"></script>
+</body>
+</html>
diff --git a/LayoutTests/js/regress/script-tests/put-by-id-transition-with-indexing-header.js b/LayoutTests/js/regress/script-tests/put-by-id-transition-with-indexing-header.js
new file mode 100644 (file)
index 0000000..37df715
--- /dev/null
@@ -0,0 +1,84 @@
+(function() {
+    function allocate() {
+        return {};
+    };
+    
+    for (var i = 0; i < 1000; ++i) {
+        var o;
+        var n = 100;
+        for (var j = 0; j < n; ++j) {
+            o = allocate();
+            o[0] = i + 0;
+            o[1] = i + 1;
+            o[2] = i + 2;
+            o[3] = i + 3;
+            o[4] = i + 4;
+            o[5] = i + 5;
+            o[6] = i + 6;
+            o[7] = i + 7;
+            o[8] = i + 8;
+            o[9] = i + 9;
+            o[10] = i + 10;
+            o[11] = i + 11;
+            o.f = j + 0;
+            o.g = j + 1;
+            o.h = j + 2;
+            o.i = j + 3;
+            o.j = j + 4;
+            o.k = j + 5;
+            o.l = j + 6;
+            o.m = j + 7;
+            o.n = j + 8;
+            o.o = j + 9;
+            o.p = j + 10;
+            o.q = j + 11;
+            o.r = j + 12;
+            o.s = j + 13;
+            o.t = j + 14;
+            o.u = j + 15;
+            o.v = j + 16;
+            o.w = j + 17;
+        }
+        
+        for (var j = 0; j < 11; ++j) {
+            if (o[j] != i + j)
+                throw "Error: bad value at o[" + j + "]: " + o[j];
+        }
+        if (o.f != n - 1 + 0)
+            throw "Error: bad value at o.f: " + o.f;
+        if (o.g != n - 1 + 1)
+            throw "Error: bad value at o.f: " + o.g;
+        if (o.h != n - 1 + 2)
+            throw "Error: bad value at o.f: " + o.h;
+        if (o.i != n - 1 + 3)
+            throw "Error: bad value at o.f: " + o.i;
+        if (o.j != n - 1 + 4)
+            throw "Error: bad value at o.f: " + o.j;
+        if (o.k != n - 1 + 5)
+            throw "Error: bad value at o.f: " + o.k;
+        if (o.l != n - 1 + 6)
+            throw "Error: bad value at o.f: " + o.l;
+        if (o.m != n - 1 + 7)
+            throw "Error: bad value at o.f: " + o.m;
+        if (o.n != n - 1 + 8)
+            throw "Error: bad value at o.f: " + o.n;
+        if (o.o != n - 1 + 9)
+            throw "Error: bad value at o.f: " + o.o;
+        if (o.p != n - 1 + 10)
+            throw "Error: bad value at o.f: " + o.p;
+        if (o.q != n - 1 + 11)
+            throw "Error: bad value at o.f: " + o.q;
+        if (o.r != n - 1 + 12)
+            throw "Error: bad value at o.f: " + o.r;
+        if (o.s != n - 1 + 13)
+            throw "Error: bad value at o.f: " + o.s;
+        if (o.t != n - 1 + 14)
+            throw "Error: bad value at o.f: " + o.t;
+        if (o.u != n - 1 + 15)
+            throw "Error: bad value at o.f: " + o.u;
+        if (o.v != n - 1 + 16)
+            throw "Error: bad value at o.f: " + o.v;
+        if (o.w != n - 1 + 17)
+            throw "Error: bad value at o.f: " + o.w;
+    }
+})();
index 8c2a00d..e8af642 100644 (file)
@@ -1,3 +1,37 @@
+2016-04-07  Filip Pizlo  <fpizlo@apple.com>
+
+        Implementing caching transition puts that need to reallocate with indexing storage
+        https://bugs.webkit.org/show_bug.cgi?id=130914
+
+        Reviewed by Saam Barati.
+
+        This enables the IC's put_by_id path to handle reallocating the out-of-line storage even if
+        the butterfly has indexing storage. Like the DFG, we do this by calling operations that
+        reallocate the butterfly. Those use JSObject API and do all of the nasty work for us, like
+        triggering a barrier.
+
+        This does a bunch of refactoring to how PolymorphicAccess makes calls. It's a lot easier to
+        do it now because the hard work is hidden under AccessGenerationState methods. This means
+        that custom accessors now share logic with put_by_id transitions.
+
+        * bytecode/PolymorphicAccess.cpp:
+        (JSC::AccessGenerationState::succeed):
+        (JSC::AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling):
+        (JSC::AccessGenerationState::preserveLiveRegistersToStackForCall):
+        (JSC::AccessGenerationState::originalCallSiteIndex):
+        (JSC::AccessGenerationState::emitExplicitExceptionHandler):
+        (JSC::AccessCase::AccessCase):
+        (JSC::AccessCase::transition):
+        (JSC::AccessCase::generate):
+        (JSC::PolymorphicAccess::regenerate):
+        * bytecode/PolymorphicAccess.h:
+        (JSC::AccessGenerationState::needsToRestoreRegistersIfException):
+        (JSC::AccessGenerationState::liveRegistersToPreserveAtExceptionHandlingCallSite):
+        * dfg/DFGOperations.cpp:
+        * dfg/DFGOperations.h:
+        * jit/JITOperations.cpp:
+        * jit/JITOperations.h:
+
 2016-04-07  Joseph Pecoraro  <pecoraro@apple.com>
 
         Remote Inspector: When disallowing remote inspection on a debuggable, a listing is still sent to debuggers
index 344246e..c78bef3 100644 (file)
@@ -70,7 +70,7 @@ void AccessGenerationState::succeed()
     success.append(jit->jump());
 }
 
-void AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling()
+void AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling(const RegisterSet& extra)
 {
     if (!m_calculatedRegistersForCallAndExceptionHandling) {
         m_calculatedRegistersForCallAndExceptionHandling = true;
@@ -82,11 +82,14 @@ void AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling()
 
         m_liveRegistersForCall = RegisterSet(m_liveRegistersToPreserveAtExceptionHandlingCallSite, allocator->usedRegisters());
         m_liveRegistersForCall.exclude(RegisterSet::registersToNotSaveForJSCall());
+        m_liveRegistersForCall.merge(extra);
     }
 }
 
-void AccessGenerationState::preserveLiveRegistersToStackForCall()
+void AccessGenerationState::preserveLiveRegistersToStackForCall(const RegisterSet& extra)
 {
+    calculateLiveRegistersForCallAndExceptionHandling(extra);
+    
     unsigned extraStackPadding = 0;
     unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(*jit, liveRegistersForCall(), extraStackPadding);
     if (m_numberOfStackBytesUsedForRegisterPreservation != std::numeric_limits<unsigned>::max())
@@ -155,6 +158,38 @@ const HandlerInfo& AccessGenerationState::originalExceptionHandler() const
 
 CallSiteIndex AccessGenerationState::originalCallSiteIndex() const { return stubInfo->callSiteIndex; }
 
+void AccessGenerationState::emitExplicitExceptionHandler()
+{
+    restoreScratch();
+    jit->copyCalleeSavesToVMCalleeSavesBuffer();
+    if (needsToRestoreRegistersIfException()) {
+        // To the JIT that produces the original exception handling
+        // call site, they will expect the OSR exit to be arrived
+        // at from genericUnwind. Therefore we must model what genericUnwind
+        // does here. I.e, set callFrameForCatch and copy callee saves.
+
+        jit->storePtr(GPRInfo::callFrameRegister, jit->vm()->addressOfCallFrameForCatch());
+        CCallHelpers::Jump jumpToOSRExitExceptionHandler = jit->jump();
+
+        // We don't need to insert a new exception handler in the table
+        // because we're doing a manual exception check here. i.e, we'll
+        // never arrive here from genericUnwind().
+        HandlerInfo originalHandler = originalExceptionHandler();
+        jit->addLinkTask(
+            [=] (LinkBuffer& linkBuffer) {
+                linkBuffer.link(jumpToOSRExitExceptionHandler, originalHandler.nativeCode);
+            });
+    } else {
+        jit->setupArguments(CCallHelpers::TrustedImmPtr(jit->vm()), GPRInfo::callFrameRegister);
+        CCallHelpers::Call lookupExceptionHandlerCall = jit->call();
+        jit->addLinkTask(
+            [=] (LinkBuffer& linkBuffer) {
+                linkBuffer.link(lookupExceptionHandlerCall, lookupExceptionHandler);
+            });
+        jit->jumpToExceptionHandler();
+    }
+}
+
 AccessCase::AccessCase()
 {
 }
@@ -243,13 +278,6 @@ std::unique_ptr<AccessCase> AccessCase::transition(
         return nullptr;
     }
 
-    // Skip optimizing the case where we need realloc, and the structure has
-    // indexing storage.
-    // FIXME: We shouldn't skip this! Implement it!
-    // https://bugs.webkit.org/show_bug.cgi?id=130914
-    if (oldStructure->couldHaveIndexingHeader())
-        return nullptr;
-
     std::unique_ptr<AccessCase> result(new AccessCase());
 
     result->m_type = Transition;
@@ -778,7 +806,6 @@ void AccessCase::generate(AccessGenerationState& state)
 
         // Stuff for custom getters/setters.
         CCallHelpers::Call operationCall;
-        CCallHelpers::Call lookupExceptionHandlerCall;
 
         // Stuff for JS getters/setters.
         CCallHelpers::DataLabelPtr addressOfLinkFunctionCheck;
@@ -790,11 +817,8 @@ void AccessCase::generate(AccessGenerationState& state)
 
         // This also does the necessary calculations of whether or not we're an
         // exception handling call site.
-        state.calculateLiveRegistersForCallAndExceptionHandling();
         state.preserveLiveRegistersToStackForCall();
 
-        // Need to make sure that whenever this call is made in the future, we remember the
-        // place that we made it from.
         jit.store32(
             CCallHelpers::TrustedImm32(state.callSiteIndexForExceptionHandlingOrOriginal().bits()),
             CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount)));
@@ -917,7 +941,7 @@ void AccessCase::generate(AccessGenerationState& state)
                 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
             state.restoreLiveRegistersFromStackForCall(isGetter());
 
-            state.callbacks.append(
+            jit.addLinkTask(
                 [=, &vm] (LinkBuffer& linkBuffer) {
                     m_rareData->callLinkInfo->setCallLocations(
                         linkBuffer.locationOfNearCall(slowPathCall),
@@ -961,6 +985,11 @@ void AccessCase::generate(AccessGenerationState& state)
             jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame);
 
             operationCall = jit.call();
+            jit.addLinkTask(
+                [=] (LinkBuffer& linkBuffer) {
+                    linkBuffer.link(operationCall, FunctionPtr(m_rareData->customAccessor.opaque));
+                });
+
             if (m_type == CustomValueGetter || m_type == CustomAccessorGetter)
                 jit.setupResults(valueRegs);
             jit.reclaimSpaceOnStackForCCall();
@@ -968,43 +997,11 @@ void AccessCase::generate(AccessGenerationState& state)
             CCallHelpers::Jump noException =
                 jit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);
 
-            bool didSetLookupExceptionHandler = false;
             state.restoreLiveRegistersFromStackForCallWithThrownException();
-            state.restoreScratch();
-            jit.copyCalleeSavesToVMCalleeSavesBuffer();
-            if (state.needsToRestoreRegistersIfException()) {
-                // To the JIT that produces the original exception handling
-                // call site, they will expect the OSR exit to be arrived
-                // at from genericUnwind. Therefore we must model what genericUnwind
-                // does here. I.e, set callFrameForCatch and copy callee saves.
-
-                jit.storePtr(GPRInfo::callFrameRegister, vm.addressOfCallFrameForCatch());
-                CCallHelpers::Jump jumpToOSRExitExceptionHandler = jit.jump();
-
-                // We don't need to insert a new exception handler in the table
-                // because we're doing a manual exception check here. i.e, we'll
-                // never arrive here from genericUnwind().
-                HandlerInfo originalHandler = state.originalExceptionHandler();
-                state.callbacks.append(
-                    [=] (LinkBuffer& linkBuffer) {
-                        linkBuffer.link(jumpToOSRExitExceptionHandler, originalHandler.nativeCode);
-                    });
-            } else {
-                jit.setupArguments(CCallHelpers::TrustedImmPtr(&vm), GPRInfo::callFrameRegister);
-                lookupExceptionHandlerCall = jit.call();
-                didSetLookupExceptionHandler = true;
-                jit.jumpToExceptionHandler();
-            }
+            state.emitExplicitExceptionHandler();
         
             noException.link(&jit);
             state.restoreLiveRegistersFromStackForCall(isGetter());
-
-            state.callbacks.append(
-                [=] (LinkBuffer& linkBuffer) {
-                    linkBuffer.link(operationCall, FunctionPtr(m_rareData->customAccessor.opaque));
-                    if (didSetLookupExceptionHandler)
-                        linkBuffer.link(lookupExceptionHandlerCall, lookupExceptionHandler);
-                });
         }
         state.succeed();
         return;
@@ -1041,7 +1038,6 @@ void AccessCase::generate(AccessGenerationState& state)
     case Transition: {
         // AccessCase::transition() should have returned null.
         RELEASE_ASSERT(GPRInfo::numberOfRegisters >= 6 || !structure()->outOfLineCapacity() || structure()->outOfLineCapacity() == newStructure()->outOfLineCapacity());
-        RELEASE_ASSERT(!structure()->couldHaveIndexingHeader());
 
         if (InferredType* type = newStructure()->inferredTypeFor(ident.impl())) {
             if (verbose)
@@ -1054,6 +1050,7 @@ void AccessCase::generate(AccessGenerationState& state)
         
         bool allocating = newStructure()->outOfLineCapacity() != structure()->outOfLineCapacity();
         bool reallocating = allocating && structure()->outOfLineCapacity();
+        bool allocatingInline = allocating && !structure()->couldHaveIndexingHeader();
 
         ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
         allocator.lock(baseGPR);
@@ -1063,12 +1060,12 @@ void AccessCase::generate(AccessGenerationState& state)
         allocator.lock(valueRegs);
         allocator.lock(scratchGPR);
 
-        GPRReg scratchGPR2 = allocator.allocateScratchGPR();
-        GPRReg scratchGPR3;
-        if (allocating)
+        GPRReg scratchGPR2 = InvalidGPRReg;
+        GPRReg scratchGPR3 = InvalidGPRReg;
+        if (allocatingInline) {
+            scratchGPR2 = allocator.allocateScratchGPR();
             scratchGPR3 = allocator.allocateScratchGPR();
-        else
-            scratchGPR3 = InvalidGPRReg;
+        }
 
         ScratchRegisterAllocator::PreservedState preservedState =
             allocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::SpaceForCCall);
@@ -1079,47 +1076,102 @@ void AccessCase::generate(AccessGenerationState& state)
 
         if (allocating) {
             size_t newSize = newStructure()->outOfLineCapacity() * sizeof(JSValue);
-            CopiedAllocator* copiedAllocator = &vm.heap.storageAllocator();
-
-            if (!reallocating) {
-                jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
-                slowPath.append(
-                    jit.branchSubPtr(
-                        CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
-                jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
-                jit.negPtr(scratchGPR);
-                jit.addPtr(
-                    CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
-                jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
-            } else {
-                size_t oldSize = structure()->outOfLineCapacity() * sizeof(JSValue);
-                ASSERT(newSize > oldSize);
             
-                jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR3);
-                jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
-                slowPath.append(
-                    jit.branchSubPtr(
-                        CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
-                jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
-                jit.negPtr(scratchGPR);
-                jit.addPtr(
-                    CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
-                jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
-                // We have scratchGPR = new storage, scratchGPR3 = old storage,
-                // scratchGPR2 = available
-                for (size_t offset = 0; offset < oldSize; offset += sizeof(void*)) {
-                    jit.loadPtr(
-                        CCallHelpers::Address(
-                            scratchGPR3,
-                            -static_cast<ptrdiff_t>(
-                                offset + sizeof(JSValue) + sizeof(void*))),
-                        scratchGPR2);
-                    jit.storePtr(
-                        scratchGPR2,
-                        CCallHelpers::Address(
-                            scratchGPR,
-                            -static_cast<ptrdiff_t>(offset + sizeof(JSValue) + sizeof(void*))));
+            if (allocatingInline) {
+                CopiedAllocator* copiedAllocator = &vm.heap.storageAllocator();
+
+                if (!reallocating) {
+                    jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
+                    slowPath.append(
+                        jit.branchSubPtr(
+                            CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
+                    jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
+                    jit.negPtr(scratchGPR);
+                    jit.addPtr(
+                        CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
+                    jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
+                } else {
+                    // Handle the case where we are reallocating (i.e. the old structure/butterfly
+                    // already had out-of-line property storage).
+                    size_t oldSize = structure()->outOfLineCapacity() * sizeof(JSValue);
+                    ASSERT(newSize > oldSize);
+            
+                    jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR3);
+                    jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
+                    slowPath.append(
+                        jit.branchSubPtr(
+                            CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
+                    jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
+                    jit.negPtr(scratchGPR);
+                    jit.addPtr(
+                        CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
+                    jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
+                    // We have scratchGPR = new storage, scratchGPR3 = old storage,
+                    // scratchGPR2 = available
+                    for (size_t offset = 0; offset < oldSize; offset += sizeof(void*)) {
+                        jit.loadPtr(
+                            CCallHelpers::Address(
+                                scratchGPR3,
+                                -static_cast<ptrdiff_t>(
+                                    offset + sizeof(JSValue) + sizeof(void*))),
+                            scratchGPR2);
+                        jit.storePtr(
+                            scratchGPR2,
+                            CCallHelpers::Address(
+                                scratchGPR,
+                                -static_cast<ptrdiff_t>(offset + sizeof(JSValue) + sizeof(void*))));
+                    }
                 }
+            } else {
+                // Handle the case where we are allocating out-of-line using an operation.
+                RegisterSet extraRegistersToPreserve;
+                extraRegistersToPreserve.set(baseGPR);
+                extraRegistersToPreserve.set(valueRegs);
+                state.preserveLiveRegistersToStackForCall(extraRegistersToPreserve);
+                
+                jit.store32(
+                    CCallHelpers::TrustedImm32(
+                        state.callSiteIndexForExceptionHandlingOrOriginal().bits()),
+                    CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount)));
+                
+                jit.makeSpaceOnStackForCCall();
+                
+                if (!reallocating) {
+                    jit.setupArgumentsWithExecState(baseGPR);
+                    
+                    CCallHelpers::Call operationCall = jit.call();
+                    jit.addLinkTask(
+                        [=] (LinkBuffer& linkBuffer) {
+                            linkBuffer.link(
+                                operationCall,
+                                FunctionPtr(operationReallocateButterflyToHavePropertyStorageWithInitialCapacity));
+                        });
+                } else {
+                    // Handle the case where we are reallocating (i.e. the old structure/butterfly
+                    // already had out-of-line property storage).
+                    jit.setupArgumentsWithExecState(
+                        baseGPR, CCallHelpers::TrustedImm32(newSize / sizeof(JSValue)));
+                    
+                    CCallHelpers::Call operationCall = jit.call();
+                    jit.addLinkTask(
+                        [=] (LinkBuffer& linkBuffer) {
+                            linkBuffer.link(
+                                operationCall,
+                                FunctionPtr(operationReallocateButterflyToGrowPropertyStorage));
+                        });
+                }
+                
+                jit.reclaimSpaceOnStackForCCall();
+                jit.move(GPRInfo::returnValueGPR, scratchGPR);
+                
+                CCallHelpers::Jump noException =
+                    jit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);
+                
+                state.restoreLiveRegistersFromStackForCallWithThrownException();
+                state.emitExplicitExceptionHandler();
+                
+                noException.link(&jit);
+                state.restoreLiveRegistersFromStackForCall();
             }
         }
 
@@ -1138,7 +1190,9 @@ void AccessCase::generate(AccessGenerationState& state)
                 CCallHelpers::Address(scratchGPR, offsetInButterfly(m_offset) * sizeof(JSValue)));
         }
         
-        if (allocating) {
+        // If we had allocated using an operation then we would have already executed the store
+        // barrier and we would have already stored the butterfly into the object.
+        if (allocatingInline) {
             CCallHelpers::Jump ownerIsRememberedOrInEden = jit.jumpIfIsRememberedOrInEden(baseGPR);
             WriteBarrierBuffer& writeBarrierBuffer = jit.vm()->heap.writeBarrierBuffer();
             jit.load32(writeBarrierBuffer.currentIndexAddress(), scratchGPR2);
@@ -1146,10 +1200,10 @@ void AccessCase::generate(AccessGenerationState& state)
                 jit.branch32(
                     CCallHelpers::AboveOrEqual, scratchGPR2,
                     CCallHelpers::TrustedImm32(writeBarrierBuffer.capacity())));
-
+            
             jit.add32(CCallHelpers::TrustedImm32(1), scratchGPR2);
             jit.store32(scratchGPR2, writeBarrierBuffer.currentIndexAddress());
-
+            
             jit.move(CCallHelpers::TrustedImmPtr(writeBarrierBuffer.buffer()), scratchGPR3);
             // We use an offset of -sizeof(void*) because we already added 1 to scratchGPR2.
             jit.storePtr(
@@ -1158,13 +1212,13 @@ void AccessCase::generate(AccessGenerationState& state)
                     scratchGPR3, scratchGPR2, CCallHelpers::ScalePtr,
                     static_cast<int32_t>(-sizeof(void*))));
             ownerIsRememberedOrInEden.link(&jit);
-        }
-        
-        // We set the new butterfly and the structure last. Doing it this way ensures that whatever
-        // we had done up to this point is forgotten if we choose to branch to slow path.
-        
-        if (allocating)
+            
+            // We set the new butterfly and the structure last. Doing it this way ensures that
+            // whatever we had done up to this point is forgotten if we choose to branch to slow
+            // path.
+            
             jit.storePtr(scratchGPR, CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()));
+        }
         
         uint32_t structureBits = bitwise_cast<uint32_t>(newStructure()->id());
         jit.store32(
@@ -1174,12 +1228,16 @@ void AccessCase::generate(AccessGenerationState& state)
         allocator.restoreReusedRegistersByPopping(jit, preservedState);
         state.succeed();
         
-        if (allocator.didReuseRegisters()) {
-            slowPath.link(&jit);
-            allocator.restoreReusedRegistersByPopping(jit, preservedState);
-            state.failAndIgnore.append(jit.jump());
+        // We will have a slow path if we were allocating without the help of an operation.
+        if (allocatingInline) {
+            if (allocator.didReuseRegisters()) {
+                slowPath.link(&jit);
+                allocator.restoreReusedRegistersByPopping(jit, preservedState);
+                state.failAndIgnore.append(jit.jump());
+            } else
+                state.failAndIgnore.append(slowPath);
         } else
-            state.failAndIgnore.append(slowPath);
+            RELEASE_ASSERT(slowPath.empty());
         return;
     }
 
@@ -1485,7 +1543,7 @@ MacroAssemblerCodePtr PolymorphicAccess::regenerate(
 
         HandlerInfo oldHandler = state.originalExceptionHandler();
         CallSiteIndex newExceptionHandlingCallSite = state.callSiteIndexForExceptionHandling();
-        state.callbacks.append(
+        jit.addLinkTask(
             [=] (LinkBuffer& linkBuffer) {
                 linkBuffer.link(jumpToOSRExitExceptionHandler, oldHandler.nativeCode);
 
@@ -1519,9 +1577,6 @@ MacroAssemblerCodePtr PolymorphicAccess::regenerate(
         failure,
         stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
     
-    for (auto callback : state.callbacks)
-        callback(linkBuffer);
-
     if (verbose)
         dataLog(*codeBlock, " ", stubInfo.codeOrigin, ": Generating polymorphic access stub for ", listDump(cases), "\n");
 
index 526ddfd..92b6881 100644 (file)
@@ -376,7 +376,6 @@ struct AccessGenerationState {
     GPRReg baseGPR { InvalidGPRReg };
     JSValueRegs valueRegs;
     GPRReg scratchGPR { InvalidGPRReg };
-    Vector<std::function<void(LinkBuffer&)>> callbacks;
     const Identifier* ident;
     std::unique_ptr<WatchpointsOnStructureStubInfo> watchpoints;
     Vector<WriteBarrier<JSCell>> weakReferences;
@@ -386,11 +385,11 @@ struct AccessGenerationState {
     void restoreScratch();
     void succeed();
 
-    void calculateLiveRegistersForCallAndExceptionHandling();
+    void calculateLiveRegistersForCallAndExceptionHandling(const RegisterSet& extra = RegisterSet());
 
-    void preserveLiveRegistersToStackForCall();
+    void preserveLiveRegistersToStackForCall(const RegisterSet& extra = RegisterSet());
 
-    void restoreLiveRegistersFromStackForCall(bool isGetter);
+    void restoreLiveRegistersFromStackForCall(bool isGetter = false);
     void restoreLiveRegistersFromStackForCallWithThrownException();
     void restoreLiveRegistersFromStackForCall(const RegisterSet& dontRestore);
 
@@ -419,6 +418,8 @@ struct AccessGenerationState {
     bool needsToRestoreRegistersIfException() const { return m_needsToRestoreRegistersIfException; }
     CallSiteIndex originalCallSiteIndex() const;
     
+    void emitExplicitExceptionHandler();
+    
 private:
     const RegisterSet& liveRegistersToPreserveAtExceptionHandlingCallSite()
     {
index 0ee4c45..e8c7ae0 100644 (file)
@@ -1112,29 +1112,6 @@ char* JIT_OPERATION operationAllocatePropertyStorage(ExecState* exec, size_t new
         Butterfly::createUninitialized(vm, 0, 0, newSize, false, 0));
 }
 
-char* JIT_OPERATION operationReallocateButterflyToHavePropertyStorageWithInitialCapacity(ExecState* exec, JSObject* object)
-{
-    VM& vm = exec->vm();
-    NativeCallFrameTracer tracer(&vm, exec);
-
-    ASSERT(!object->structure()->outOfLineCapacity());
-    DeferGC deferGC(vm.heap);
-    Butterfly* result = object->growOutOfLineStorage(vm, 0, initialOutOfLineCapacity);
-    object->setButterflyWithoutChangingStructure(vm, result);
-    return reinterpret_cast<char*>(result);
-}
-
-char* JIT_OPERATION operationReallocateButterflyToGrowPropertyStorage(ExecState* exec, JSObject* object, size_t newSize)
-{
-    VM& vm = exec->vm();
-    NativeCallFrameTracer tracer(&vm, exec);
-
-    DeferGC deferGC(vm.heap);
-    Butterfly* result = object->growOutOfLineStorage(vm, object->structure()->outOfLineCapacity(), newSize);
-    object->setButterflyWithoutChangingStructure(vm, result);
-    return reinterpret_cast<char*>(result);
-}
-
 char* JIT_OPERATION operationEnsureInt32(ExecState* exec, JSCell* cell)
 {
     VM& vm = exec->vm();
index 6e40caf..3f4e27a 100644 (file)
@@ -124,8 +124,6 @@ JSCell* JIT_OPERATION operationTypeOfObject(ExecState*, JSGlobalObject*, JSCell*
 int32_t JIT_OPERATION operationTypeOfObjectAsTypeofType(ExecState*, JSGlobalObject*, JSCell*) WTF_INTERNAL;
 char* JIT_OPERATION operationAllocatePropertyStorageWithInitialCapacity(ExecState*) WTF_INTERNAL;
 char* JIT_OPERATION operationAllocatePropertyStorage(ExecState*, size_t newSize) WTF_INTERNAL;
-char* JIT_OPERATION operationReallocateButterflyToHavePropertyStorageWithInitialCapacity(ExecState*, JSObject*) WTF_INTERNAL;
-char* JIT_OPERATION operationReallocateButterflyToGrowPropertyStorage(ExecState*, JSObject*, size_t newSize) WTF_INTERNAL;
 char* JIT_OPERATION operationEnsureInt32(ExecState*, JSCell*);
 char* JIT_OPERATION operationEnsureDouble(ExecState*, JSCell*);
 char* JIT_OPERATION operationEnsureContiguous(ExecState*, JSCell*);
index ec73c1c..1d746cb 100644 (file)
@@ -2030,6 +2030,29 @@ void JIT_OPERATION operationThrow(ExecState* exec, EncodedJSValue encodedExcepti
     genericUnwind(vm, exec);
 }
 
+char* JIT_OPERATION operationReallocateButterflyToHavePropertyStorageWithInitialCapacity(ExecState* exec, JSObject* object)
+{
+    VM& vm = exec->vm();
+    NativeCallFrameTracer tracer(&vm, exec);
+
+    ASSERT(!object->structure()->outOfLineCapacity());
+    DeferGC deferGC(vm.heap);
+    Butterfly* result = object->growOutOfLineStorage(vm, 0, initialOutOfLineCapacity);
+    object->setButterflyWithoutChangingStructure(vm, result);
+    return reinterpret_cast<char*>(result);
+}
+
+char* JIT_OPERATION operationReallocateButterflyToGrowPropertyStorage(ExecState* exec, JSObject* object, size_t newSize)
+{
+    VM& vm = exec->vm();
+    NativeCallFrameTracer tracer(&vm, exec);
+
+    DeferGC deferGC(vm.heap);
+    Butterfly* result = object->growOutOfLineStorage(vm, object->structure()->outOfLineCapacity(), newSize);
+    object->setButterflyWithoutChangingStructure(vm, result);
+    return reinterpret_cast<char*>(result);
+}
+
 void JIT_OPERATION operationFlushWriteBarrierBuffer(ExecState* exec, JSCell* cell)
 {
     VM* vm = &exec->vm();
index 8e82b3b..d686ab0 100644 (file)
@@ -377,6 +377,9 @@ char* JIT_OPERATION operationSwitchStringWithUnknownKeyType(ExecState*, EncodedJ
 EncodedJSValue JIT_OPERATION operationGetFromScope(ExecState*, Instruction* bytecodePC) WTF_INTERNAL;
 void JIT_OPERATION operationPutToScope(ExecState*, Instruction* bytecodePC) WTF_INTERNAL;
 
+char* JIT_OPERATION operationReallocateButterflyToHavePropertyStorageWithInitialCapacity(ExecState*, JSObject*) WTF_INTERNAL;
+char* JIT_OPERATION operationReallocateButterflyToGrowPropertyStorage(ExecState*, JSObject*, size_t newSize) WTF_INTERNAL;
+
 void JIT_OPERATION operationFlushWriteBarrierBuffer(ExecState*, JSCell*);
 void JIT_OPERATION operationWriteBarrier(ExecState*, JSCell*, JSCell*);
 void JIT_OPERATION operationUnconditionalWriteBarrier(ExecState*, JSCell*);