Add WTF::move()
[WebKit-https.git] / Source / JavaScriptCore / jit / Repatch.cpp
1 /*
2  * Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "Repatch.h"
28
29 #if ENABLE(JIT)
30
31 #include "AccessorCallJITStubRoutine.h"
32 #include "CCallHelpers.h"
33 #include "DFGOperations.h"
34 #include "DFGSpeculativeJIT.h"
35 #include "FTLThunks.h"
36 #include "GCAwareJITStubRoutine.h"
37 #include "GetterSetter.h"
38 #include "JIT.h"
39 #include "JITInlines.h"
40 #include "LinkBuffer.h"
41 #include "JSCInlines.h"
42 #include "PolymorphicGetByIdList.h"
43 #include "PolymorphicPutByIdList.h"
44 #include "RegExpMatchesArray.h"
45 #include "RepatchBuffer.h"
46 #include "ScratchRegisterAllocator.h"
47 #include "StackAlignment.h"
48 #include "StructureRareDataInlines.h"
49 #include "StructureStubClearingWatchpoint.h"
50 #include "ThunkGenerators.h"
51 #include <wtf/StringPrintStream.h>
52
53 namespace JSC {
54
55 // Beware: in this code, it is not safe to assume anything about the following registers
56 // that would ordinarily have well-known values:
57 // - tagTypeNumberRegister
58 // - tagMaskRegister
59
60 static FunctionPtr readCallTarget(RepatchBuffer& repatchBuffer, CodeLocationCall call)
61 {
62     FunctionPtr result = MacroAssembler::readCallTarget(call);
63 #if ENABLE(FTL_JIT)
64     CodeBlock* codeBlock = repatchBuffer.codeBlock();
65     if (codeBlock->jitType() == JITCode::FTLJIT) {
66         return FunctionPtr(codeBlock->vm()->ftlThunks->keyForSlowPathCallThunk(
67             MacroAssemblerCodePtr::createFromExecutableAddress(
68                 result.executableAddress())).callTarget());
69     }
70 #else
71     UNUSED_PARAM(repatchBuffer);
72 #endif // ENABLE(FTL_JIT)
73     return result;
74 }
75
76 static void repatchCall(RepatchBuffer& repatchBuffer, CodeLocationCall call, FunctionPtr newCalleeFunction)
77 {
78 #if ENABLE(FTL_JIT)
79     CodeBlock* codeBlock = repatchBuffer.codeBlock();
80     if (codeBlock->jitType() == JITCode::FTLJIT) {
81         VM& vm = *codeBlock->vm();
82         FTL::Thunks& thunks = *vm.ftlThunks;
83         FTL::SlowPathCallKey key = thunks.keyForSlowPathCallThunk(
84             MacroAssemblerCodePtr::createFromExecutableAddress(
85                 MacroAssembler::readCallTarget(call).executableAddress()));
86         key = key.withCallTarget(newCalleeFunction.executableAddress());
87         newCalleeFunction = FunctionPtr(
88             thunks.getSlowPathCallThunk(vm, key).code().executableAddress());
89     }
90 #endif // ENABLE(FTL_JIT)
91     repatchBuffer.relink(call, newCalleeFunction);
92 }
93
94 static void repatchCall(CodeBlock* codeblock, CodeLocationCall call, FunctionPtr newCalleeFunction)
95 {
96     RepatchBuffer repatchBuffer(codeblock);
97     repatchCall(repatchBuffer, call, newCalleeFunction);
98 }
99
100 static void repatchByIdSelfAccess(VM& vm, CodeBlock* codeBlock, StructureStubInfo& stubInfo, Structure* structure, const Identifier& propertyName, PropertyOffset offset,
101     const FunctionPtr &slowPathFunction, bool compact)
102 {
103     if (structure->typeInfo().newImpurePropertyFiresWatchpoints())
104         vm.registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));
105
106     RepatchBuffer repatchBuffer(codeBlock);
107
108     // Only optimize once!
109     repatchCall(repatchBuffer, stubInfo.callReturnLocation, slowPathFunction);
110
111     // Patch the structure check & the offset of the load.
112     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(-(intptr_t)stubInfo.patch.deltaCheckImmToCall), bitwise_cast<int32_t>(structure->id()));
113     repatchBuffer.setLoadInstructionIsActive(stubInfo.callReturnLocation.convertibleLoadAtOffset(stubInfo.patch.deltaCallToStorageLoad), isOutOfLineOffset(offset));
114 #if USE(JSVALUE64)
115     if (compact)
116         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToLoadOrStore), offsetRelativeToPatchedStorage(offset));
117     else
118         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToLoadOrStore), offsetRelativeToPatchedStorage(offset));
119 #elif USE(JSVALUE32_64)
120     if (compact) {
121         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
122         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
123     } else {
124         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
125         repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
126     }
127 #endif
128 }
129
130 static void addStructureTransitionCheck(
131     JSCell* object, Structure* structure, CodeBlock* codeBlock, StructureStubInfo& stubInfo,
132     MacroAssembler& jit, MacroAssembler::JumpList& failureCases, GPRReg scratchGPR)
133 {
134     if (object->structure() == structure && structure->transitionWatchpointSetIsStillValid()) {
135         structure->addTransitionWatchpoint(stubInfo.addWatchpoint(codeBlock));
136         if (!ASSERT_DISABLED) {
137             // If we execute this code, the object must have the structure we expect. Assert
138             // this in debug modes.
139             jit.move(MacroAssembler::TrustedImmPtr(object), scratchGPR);
140             MacroAssembler::Jump ok = branchStructure(
141                 jit,
142                 MacroAssembler::Equal,
143                 MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),
144                 structure);
145             jit.abortWithReason(RepatchIneffectiveWatchpoint);
146             ok.link(&jit);
147         }
148         return;
149     }
150     
151     jit.move(MacroAssembler::TrustedImmPtr(object), scratchGPR);
152     failureCases.append(
153         branchStructure(jit,
154             MacroAssembler::NotEqual,
155             MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),
156             structure));
157 }
158
159 static void addStructureTransitionCheck(
160     JSValue prototype, CodeBlock* codeBlock, StructureStubInfo& stubInfo,
161     MacroAssembler& jit, MacroAssembler::JumpList& failureCases, GPRReg scratchGPR)
162 {
163     if (prototype.isNull())
164         return;
165     
166     ASSERT(prototype.isCell());
167     
168     addStructureTransitionCheck(
169         prototype.asCell(), prototype.asCell()->structure(), codeBlock, stubInfo, jit,
170         failureCases, scratchGPR);
171 }
172
173 static void replaceWithJump(RepatchBuffer& repatchBuffer, StructureStubInfo& stubInfo, const MacroAssemblerCodePtr target)
174 {
175     if (MacroAssembler::canJumpReplacePatchableBranch32WithPatch()) {
176         repatchBuffer.replaceWithJump(
177             RepatchBuffer::startOfPatchableBranch32WithPatchOnAddress(
178                 stubInfo.callReturnLocation.dataLabel32AtOffset(
179                     -(intptr_t)stubInfo.patch.deltaCheckImmToCall)),
180             CodeLocationLabel(target));
181         return;
182     }
183     
184     repatchBuffer.relink(
185         stubInfo.callReturnLocation.jumpAtOffset(
186             stubInfo.patch.deltaCallToJump),
187         CodeLocationLabel(target));
188 }
189
190 static void emitRestoreScratch(MacroAssembler& stubJit, bool needToRestoreScratch, GPRReg scratchGPR, MacroAssembler::Jump& success, MacroAssembler::Jump& fail, MacroAssembler::JumpList failureCases)
191 {
192     if (needToRestoreScratch) {
193         stubJit.popToRestore(scratchGPR);
194         
195         success = stubJit.jump();
196         
197         // link failure cases here, so we can pop scratchGPR, and then jump back.
198         failureCases.link(&stubJit);
199         
200         stubJit.popToRestore(scratchGPR);
201         
202         fail = stubJit.jump();
203         return;
204     }
205     
206     success = stubJit.jump();
207 }
208
209 static void linkRestoreScratch(LinkBuffer& patchBuffer, bool needToRestoreScratch, MacroAssembler::Jump success, MacroAssembler::Jump fail, MacroAssembler::JumpList failureCases, CodeLocationLabel successLabel, CodeLocationLabel slowCaseBegin)
210 {
211     patchBuffer.link(success, successLabel);
212         
213     if (needToRestoreScratch) {
214         patchBuffer.link(fail, slowCaseBegin);
215         return;
216     }
217     
218     // link failure cases directly back to normal path
219     patchBuffer.link(failureCases, slowCaseBegin);
220 }
221
222 static void linkRestoreScratch(LinkBuffer& patchBuffer, bool needToRestoreScratch, StructureStubInfo& stubInfo, MacroAssembler::Jump success, MacroAssembler::Jump fail, MacroAssembler::JumpList failureCases)
223 {
224     linkRestoreScratch(patchBuffer, needToRestoreScratch, success, fail, failureCases, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone), stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
225 }
226
227 enum ByIdStubKind {
228     GetValue,
229     CallGetter,
230     CallCustomGetter,
231     CallSetter,
232     CallCustomSetter
233 };
234
235 static const char* toString(ByIdStubKind kind)
236 {
237     switch (kind) {
238     case GetValue:
239         return "GetValue";
240     case CallGetter:
241         return "CallGetter";
242     case CallCustomGetter:
243         return "CallCustomGetter";
244     case CallSetter:
245         return "CallSetter";
246     case CallCustomSetter:
247         return "CallCustomSetter";
248     default:
249         RELEASE_ASSERT_NOT_REACHED();
250         return nullptr;
251     }
252 }
253
254 static ByIdStubKind kindFor(const PropertySlot& slot)
255 {
256     if (slot.isCacheableValue())
257         return GetValue;
258     if (slot.isCacheableCustom())
259         return CallCustomGetter;
260     RELEASE_ASSERT(slot.isCacheableGetter());
261     return CallGetter;
262 }
263
264 static FunctionPtr customFor(const PropertySlot& slot)
265 {
266     if (!slot.isCacheableCustom())
267         return FunctionPtr();
268     return FunctionPtr(slot.customGetter());
269 }
270
271 static ByIdStubKind kindFor(const PutPropertySlot& slot)
272 {
273     RELEASE_ASSERT(!slot.isCacheablePut());
274     if (slot.isCacheableSetter())
275         return CallSetter;
276     RELEASE_ASSERT(slot.isCacheableCustom());
277     return CallCustomSetter;
278 }
279
280 static FunctionPtr customFor(const PutPropertySlot& slot)
281 {
282     if (!slot.isCacheableCustom())
283         return FunctionPtr();
284     return FunctionPtr(slot.customSetter());
285 }
286
287 static void generateByIdStub(
288     ExecState* exec, ByIdStubKind kind, const Identifier& propertyName,
289     FunctionPtr custom, StructureStubInfo& stubInfo, StructureChain* chain, size_t count,
290     PropertyOffset offset, Structure* structure, bool loadTargetFromProxy, WatchpointSet* watchpointSet,
291     CodeLocationLabel successLabel, CodeLocationLabel slowCaseLabel, RefPtr<JITStubRoutine>& stubRoutine)
292 {
293     VM* vm = &exec->vm();
294     GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
295     JSValueRegs valueRegs = JSValueRegs(
296 #if USE(JSVALUE32_64)
297         static_cast<GPRReg>(stubInfo.patch.valueTagGPR),
298 #endif
299         static_cast<GPRReg>(stubInfo.patch.valueGPR));
300     GPRReg scratchGPR = TempRegisterSet(stubInfo.patch.usedRegisters).getFreeGPR();
301     bool needToRestoreScratch = scratchGPR == InvalidGPRReg;
302     RELEASE_ASSERT(!needToRestoreScratch || kind == GetValue);
303     
304     CCallHelpers stubJit(&exec->vm(), exec->codeBlock());
305     if (needToRestoreScratch) {
306         scratchGPR = AssemblyHelpers::selectScratchGPR(
307             baseGPR, valueRegs.tagGPR(), valueRegs.payloadGPR());
308         stubJit.pushToSave(scratchGPR);
309         needToRestoreScratch = true;
310     }
311     
312     MacroAssembler::JumpList failureCases;
313
314     GPRReg baseForGetGPR;
315     if (loadTargetFromProxy) {
316         baseForGetGPR = valueRegs.payloadGPR();
317         failureCases.append(stubJit.branch8(
318             MacroAssembler::NotEqual, 
319             MacroAssembler::Address(baseGPR, JSCell::typeInfoTypeOffset()), 
320             MacroAssembler::TrustedImm32(PureForwardingProxyType)));
321
322         stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSProxy::targetOffset()), scratchGPR);
323         
324         failureCases.append(branchStructure(stubJit,
325             MacroAssembler::NotEqual, 
326             MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),
327             structure));
328     } else {
329         baseForGetGPR = baseGPR;
330
331         failureCases.append(branchStructure(stubJit,
332             MacroAssembler::NotEqual, 
333             MacroAssembler::Address(baseForGetGPR, JSCell::structureIDOffset()), 
334             structure));
335     }
336
337     CodeBlock* codeBlock = exec->codeBlock();
338     if (structure->typeInfo().newImpurePropertyFiresWatchpoints())
339         vm->registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));
340
341     if (watchpointSet)
342         watchpointSet->add(stubInfo.addWatchpoint(codeBlock));
343
344     Structure* currStructure = structure;
345     JSObject* protoObject = 0;
346     if (chain) {
347         WriteBarrier<Structure>* it = chain->head();
348         for (unsigned i = 0; i < count; ++i, ++it) {
349             protoObject = asObject(currStructure->prototypeForLookup(exec));
350             Structure* protoStructure = protoObject->structure();
351             if (protoStructure->typeInfo().newImpurePropertyFiresWatchpoints())
352                 vm->registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));
353             addStructureTransitionCheck(
354                 protoObject, protoStructure, codeBlock, stubInfo, stubJit,
355                 failureCases, scratchGPR);
356             currStructure = it->get();
357         }
358     }
359     
360     GPRReg baseForAccessGPR;
361     if (chain) {
362         // We could have clobbered scratchGPR earlier, so we have to reload from baseGPR to get the target.
363         if (loadTargetFromProxy)
364             stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSProxy::targetOffset()), baseForGetGPR);
365         stubJit.move(MacroAssembler::TrustedImmPtr(protoObject), scratchGPR);
366         baseForAccessGPR = scratchGPR;
367     } else {
368         // For proxy objects, we need to do all the Structure checks before moving the baseGPR into 
369         // baseForGetGPR because if we fail any of the checks then we would have the wrong value in baseGPR
370         // on the slow path.
371         if (loadTargetFromProxy)
372             stubJit.move(scratchGPR, baseForGetGPR);
373         baseForAccessGPR = baseForGetGPR;
374     }
375
376     GPRReg loadedValueGPR = InvalidGPRReg;
377     if (kind != CallCustomGetter && kind != CallCustomSetter) {
378         if (kind == GetValue)
379             loadedValueGPR = valueRegs.payloadGPR();
380         else
381             loadedValueGPR = scratchGPR;
382         
383         GPRReg storageGPR;
384         if (isInlineOffset(offset))
385             storageGPR = baseForAccessGPR;
386         else {
387             stubJit.loadPtr(MacroAssembler::Address(baseForAccessGPR, JSObject::butterflyOffset()), loadedValueGPR);
388             storageGPR = loadedValueGPR;
389         }
390         
391 #if USE(JSVALUE64)
392         stubJit.load64(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset)), loadedValueGPR);
393 #else
394         if (kind == GetValue)
395             stubJit.load32(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset) + TagOffset), valueRegs.tagGPR());
396         stubJit.load32(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset) + PayloadOffset), loadedValueGPR);
397 #endif
398     }
399
400     // Stuff for custom getters.
401     MacroAssembler::Call operationCall;
402     MacroAssembler::Call handlerCall;
403
404     // Stuff for JS getters.
405     MacroAssembler::DataLabelPtr addressOfLinkFunctionCheck;
406     MacroAssembler::Call fastPathCall;
407     MacroAssembler::Call slowPathCall;
408     std::unique_ptr<CallLinkInfo> callLinkInfo;
409
410     MacroAssembler::Jump success, fail;
411     if (kind != GetValue) {
412         // Need to make sure that whenever this call is made in the future, we remember the
413         // place that we made it from. It just so happens to be the place that we are at
414         // right now!
415         stubJit.store32(MacroAssembler::TrustedImm32(exec->locationAsRawBits()),
416             CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount)));
417
418         if (kind == CallGetter || kind == CallSetter) {
419             // Create a JS call using a JS call inline cache. Assume that:
420             //
421             // - SP is aligned and represents the extent of the calling compiler's stack usage.
422             //
423             // - FP is set correctly (i.e. it points to the caller's call frame header).
424             //
425             // - SP - FP is an aligned difference.
426             //
427             // - Any byte between FP (exclusive) and SP (inclusive) could be live in the calling
428             //   code.
429             //
430             // Therefore, we temporarily grow the stack for the purpose of the call and then
431             // shrink it after.
432             
433             callLinkInfo = std::make_unique<CallLinkInfo>();
434             callLinkInfo->callType = CallLinkInfo::Call;
435             callLinkInfo->codeOrigin = stubInfo.codeOrigin;
436             callLinkInfo->calleeGPR = loadedValueGPR;
437             
438             MacroAssembler::JumpList done;
439             
440             // There is a 'this' argument but nothing else.
441             unsigned numberOfParameters = 1;
442             // ... unless we're calling a setter.
443             if (kind == CallSetter)
444                 numberOfParameters++;
445             
446             // Get the accessor; if there ain't one then the result is jsUndefined().
447             if (kind == CallSetter) {
448                 stubJit.loadPtr(
449                     MacroAssembler::Address(loadedValueGPR, GetterSetter::offsetOfSetter()),
450                     loadedValueGPR);
451             } else {
452                 stubJit.loadPtr(
453                     MacroAssembler::Address(loadedValueGPR, GetterSetter::offsetOfGetter()),
454                     loadedValueGPR);
455             }
456             MacroAssembler::Jump returnUndefined = stubJit.branchTestPtr(
457                 MacroAssembler::Zero, loadedValueGPR);
458             
459             unsigned numberOfRegsForCall =
460                 JSStack::CallFrameHeaderSize + numberOfParameters;
461             
462             unsigned numberOfBytesForCall =
463                 numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);
464             
465             unsigned alignedNumberOfBytesForCall =
466                 WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);
467             
468             stubJit.subPtr(
469                 MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),
470                 MacroAssembler::stackPointerRegister);
471             
472             MacroAssembler::Address calleeFrame = MacroAssembler::Address(
473                 MacroAssembler::stackPointerRegister,
474                 -static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
475             
476             stubJit.store32(
477                 MacroAssembler::TrustedImm32(numberOfParameters),
478                 calleeFrame.withOffset(
479                     JSStack::ArgumentCount * sizeof(Register) + PayloadOffset));
480             
481             stubJit.storeCell(
482                 loadedValueGPR, calleeFrame.withOffset(JSStack::Callee * sizeof(Register)));
483
484             stubJit.storeCell(
485                 baseForGetGPR,
486                 calleeFrame.withOffset(
487                     virtualRegisterForArgument(0).offset() * sizeof(Register)));
488             
489             if (kind == CallSetter) {
490                 stubJit.storeValue(
491                     valueRegs,
492                     calleeFrame.withOffset(
493                         virtualRegisterForArgument(1).offset() * sizeof(Register)));
494             }
495             
496             MacroAssembler::Jump slowCase = stubJit.branchPtrWithPatch(
497                 MacroAssembler::NotEqual, loadedValueGPR, addressOfLinkFunctionCheck,
498                 MacroAssembler::TrustedImmPtr(0));
499             
500             // loadedValueGPR is already burned. We can reuse it. From here on we assume that
501             // any volatile register will be clobbered anyway.
502             stubJit.loadPtr(
503                 MacroAssembler::Address(loadedValueGPR, JSFunction::offsetOfScopeChain()),
504                 loadedValueGPR);
505             stubJit.storeCell(
506                 loadedValueGPR, calleeFrame.withOffset(JSStack::ScopeChain * sizeof(Register)));
507             fastPathCall = stubJit.nearCall();
508             
509             stubJit.addPtr(
510                 MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),
511                 MacroAssembler::stackPointerRegister);
512             if (kind == CallGetter)
513                 stubJit.setupResults(valueRegs);
514             
515             done.append(stubJit.jump());
516             slowCase.link(&stubJit);
517             
518             stubJit.move(loadedValueGPR, GPRInfo::regT0);
519 #if USE(JSVALUE32_64)
520             stubJit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), GPRInfo::regT1);
521 #endif
522             stubJit.move(MacroAssembler::TrustedImmPtr(callLinkInfo.get()), GPRInfo::regT2);
523             slowPathCall = stubJit.nearCall();
524             
525             stubJit.addPtr(
526                 MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),
527                 MacroAssembler::stackPointerRegister);
528             if (kind == CallGetter)
529                 stubJit.setupResults(valueRegs);
530             
531             done.append(stubJit.jump());
532             returnUndefined.link(&stubJit);
533             
534             if (kind == CallGetter)
535                 stubJit.moveTrustedValue(jsUndefined(), valueRegs);
536             
537             done.link(&stubJit);
538         } else {
539             // getter: EncodedJSValue (*GetValueFunc)(ExecState*, JSObject* slotBase, EncodedJSValue thisValue, PropertyName);
540             // setter: void (*PutValueFunc)(ExecState*, JSObject* base, EncodedJSValue thisObject, EncodedJSValue value);
541 #if USE(JSVALUE64)
542             if (kind == CallCustomGetter)
543                 stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImmPtr(propertyName.impl()));
544             else
545                 stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, valueRegs.gpr());
546 #else
547             if (kind == CallCustomGetter)
548                 stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImm32(JSValue::CellTag), MacroAssembler::TrustedImmPtr(propertyName.impl()));
549             else
550                 stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImm32(JSValue::CellTag), valueRegs.payloadGPR(), valueRegs.tagGPR());
551 #endif
552             stubJit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);
553
554             operationCall = stubJit.call();
555             if (kind == CallCustomGetter)
556                 stubJit.setupResults(valueRegs);
557             MacroAssembler::Jump noException = stubJit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);
558             
559             stubJit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
560             handlerCall = stubJit.call();
561             stubJit.jumpToExceptionHandler();
562             
563             noException.link(&stubJit);
564         }
565     }
566     emitRestoreScratch(stubJit, needToRestoreScratch, scratchGPR, success, fail, failureCases);
567     
568     LinkBuffer patchBuffer(*vm, &stubJit, exec->codeBlock());
569     
570     linkRestoreScratch(patchBuffer, needToRestoreScratch, success, fail, failureCases, successLabel, slowCaseLabel);
571     if (kind == CallCustomGetter || kind == CallCustomSetter) {
572         patchBuffer.link(operationCall, custom);
573         patchBuffer.link(handlerCall, lookupExceptionHandler);
574     } else if (kind == CallGetter || kind == CallSetter) {
575         callLinkInfo->hotPathOther = patchBuffer.locationOfNearCall(fastPathCall);
576         callLinkInfo->hotPathBegin = patchBuffer.locationOf(addressOfLinkFunctionCheck);
577         callLinkInfo->callReturnLocation = patchBuffer.locationOfNearCall(slowPathCall);
578
579         ThunkGenerator generator = linkThunkGeneratorFor(
580             CodeForCall, RegisterPreservationNotRequired);
581         patchBuffer.link(
582             slowPathCall, CodeLocationLabel(vm->getCTIStub(generator).code()));
583     }
584     
585     MacroAssemblerCodeRef code = FINALIZE_CODE_FOR(
586         exec->codeBlock(), patchBuffer,
587         ("%s access stub for %s, return point %p",
588             toString(kind), toCString(*exec->codeBlock()).data(),
589             successLabel.executableAddress()));
590     
591     if (kind == CallGetter || kind == CallSetter)
592         stubRoutine = adoptRef(new AccessorCallJITStubRoutine(code, *vm, WTF::move(callLinkInfo)));
593     else
594         stubRoutine = createJITStubRoutine(code, *vm, codeBlock->ownerExecutable(), true);
595 }
596
597 enum InlineCacheAction {
598     GiveUpOnCache,
599     RetryCacheLater,
600     AttemptToCache
601 };
602
603 static InlineCacheAction actionForCell(VM& vm, JSCell* cell)
604 {
605     Structure* structure = cell->structure(vm);
606
607     TypeInfo typeInfo = structure->typeInfo();
608     if (typeInfo.prohibitsPropertyCaching())
609         return GiveUpOnCache;
610
611     if (structure->isUncacheableDictionary()) {
612         if (structure->hasBeenFlattenedBefore())
613             return GiveUpOnCache;
614         // Flattening could have changed the offset, so return early for another try.
615         asObject(cell)->flattenDictionaryObject(vm);
616         return RetryCacheLater;
617     }
618     ASSERT(!structure->isUncacheableDictionary());
619     
620     if (typeInfo.hasImpureGetOwnPropertySlot() && !typeInfo.newImpurePropertyFiresWatchpoints())
621         return GiveUpOnCache;
622
623     return AttemptToCache;
624 }
625
626 static InlineCacheAction tryCacheGetByID(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)
627 {
628     if (Options::forceICFailure())
629         return GiveUpOnCache;
630     
631     // FIXME: Write a test that proves we need to check for recursion here just
632     // like the interpreter does, then add a check for recursion.
633
634     CodeBlock* codeBlock = exec->codeBlock();
635     VM* vm = &exec->vm();
636
637     if ((isJSArray(baseValue) || isRegExpMatchesArray(baseValue) || isJSString(baseValue)) && propertyName == exec->propertyNames().length) {
638         GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
639 #if USE(JSVALUE32_64)
640         GPRReg resultTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);
641 #endif
642         GPRReg resultGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);
643
644         MacroAssembler stubJit;
645
646         if (isJSArray(baseValue) || isRegExpMatchesArray(baseValue)) {
647             GPRReg scratchGPR = TempRegisterSet(stubInfo.patch.usedRegisters).getFreeGPR();
648             bool needToRestoreScratch = false;
649
650             if (scratchGPR == InvalidGPRReg) {
651 #if USE(JSVALUE64)
652                 scratchGPR = AssemblyHelpers::selectScratchGPR(baseGPR, resultGPR);
653 #else
654                 scratchGPR = AssemblyHelpers::selectScratchGPR(baseGPR, resultGPR, resultTagGPR);
655 #endif
656                 stubJit.pushToSave(scratchGPR);
657                 needToRestoreScratch = true;
658             }
659
660             MacroAssembler::JumpList failureCases;
661
662             stubJit.load8(MacroAssembler::Address(baseGPR, JSCell::indexingTypeOffset()), scratchGPR);
663             failureCases.append(stubJit.branchTest32(MacroAssembler::Zero, scratchGPR, MacroAssembler::TrustedImm32(IsArray)));
664             failureCases.append(stubJit.branchTest32(MacroAssembler::Zero, scratchGPR, MacroAssembler::TrustedImm32(IndexingShapeMask)));
665
666             stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);
667             stubJit.load32(MacroAssembler::Address(scratchGPR, ArrayStorage::lengthOffset()), scratchGPR);
668             failureCases.append(stubJit.branch32(MacroAssembler::LessThan, scratchGPR, MacroAssembler::TrustedImm32(0)));
669
670             stubJit.move(scratchGPR, resultGPR);
671 #if USE(JSVALUE64)
672             stubJit.or64(AssemblyHelpers::TrustedImm64(TagTypeNumber), resultGPR);
673 #elif USE(JSVALUE32_64)
674             stubJit.move(AssemblyHelpers::TrustedImm32(0xffffffff), resultTagGPR); // JSValue::Int32Tag
675 #endif
676
677             MacroAssembler::Jump success, fail;
678
679             emitRestoreScratch(stubJit, needToRestoreScratch, scratchGPR, success, fail, failureCases);
680             
681             LinkBuffer patchBuffer(*vm, &stubJit, codeBlock);
682
683             linkRestoreScratch(patchBuffer, needToRestoreScratch, stubInfo, success, fail, failureCases);
684
685             stubInfo.stubRoutine = FINALIZE_CODE_FOR_STUB(
686                 exec->codeBlock(), patchBuffer,
687                 ("GetById array length stub for %s, return point %p",
688                     toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(
689                         stubInfo.patch.deltaCallToDone).executableAddress()));
690
691             RepatchBuffer repatchBuffer(codeBlock);
692             replaceWithJump(repatchBuffer, stubInfo, stubInfo.stubRoutine->code().code());
693             repatchCall(repatchBuffer, stubInfo.callReturnLocation, operationGetById);
694
695             return RetryCacheLater;
696         }
697
698         // String.length case
699         MacroAssembler::Jump failure = stubJit.branch8(MacroAssembler::NotEqual, MacroAssembler::Address(baseGPR, JSCell::typeInfoTypeOffset()), MacroAssembler::TrustedImm32(StringType));
700
701         stubJit.load32(MacroAssembler::Address(baseGPR, JSString::offsetOfLength()), resultGPR);
702
703 #if USE(JSVALUE64)
704         stubJit.or64(AssemblyHelpers::TrustedImm64(TagTypeNumber), resultGPR);
705 #elif USE(JSVALUE32_64)
706         stubJit.move(AssemblyHelpers::TrustedImm32(0xffffffff), resultTagGPR); // JSValue::Int32Tag
707 #endif
708
709         MacroAssembler::Jump success = stubJit.jump();
710
711         LinkBuffer patchBuffer(*vm, &stubJit, codeBlock);
712
713         patchBuffer.link(success, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));
714         patchBuffer.link(failure, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
715
716         stubInfo.stubRoutine = FINALIZE_CODE_FOR_STUB(
717             exec->codeBlock(), patchBuffer,
718             ("GetById string length stub for %s, return point %p",
719                 toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(
720                     stubInfo.patch.deltaCallToDone).executableAddress()));
721
722         RepatchBuffer repatchBuffer(codeBlock);
723         replaceWithJump(repatchBuffer, stubInfo, stubInfo.stubRoutine->code().code());
724         repatchCall(repatchBuffer, stubInfo.callReturnLocation, operationGetById);
725
726         return RetryCacheLater;
727     }
728
729     // FIXME: Cache property access for immediates.
730     if (!baseValue.isCell())
731         return GiveUpOnCache;
732     JSCell* baseCell = baseValue.asCell();
733     Structure* structure = baseCell->structure();
734     if (!slot.isCacheable())
735         return GiveUpOnCache;
736
737     InlineCacheAction action = actionForCell(*vm, baseCell);
738     if (action != AttemptToCache)
739         return action;
740
741     // Optimize self access.
742     if (slot.slotBase() == baseValue
743         && slot.isCacheableValue()
744         && !slot.watchpointSet()
745         && MacroAssembler::isCompactPtrAlignedAddressOffset(maxOffsetRelativeToPatchedStorage(slot.cachedOffset()))) {
746             repatchByIdSelfAccess(*vm, codeBlock, stubInfo, structure, propertyName, slot.cachedOffset(), operationGetByIdBuildList, true);
747             stubInfo.initGetByIdSelf(*vm, codeBlock->ownerExecutable(), structure);
748             return RetryCacheLater;
749     }
750
751     repatchCall(codeBlock, stubInfo.callReturnLocation, operationGetByIdBuildList);
752     return RetryCacheLater;
753 }
754
755 void repatchGetByID(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)
756 {
757     GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);
758     
759     if (tryCacheGetByID(exec, baseValue, propertyName, slot, stubInfo) == GiveUpOnCache)
760         repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, operationGetById);
761 }
762
763 static void patchJumpToGetByIdStub(CodeBlock* codeBlock, StructureStubInfo& stubInfo, JITStubRoutine* stubRoutine)
764 {
765     RELEASE_ASSERT(stubInfo.accessType == access_get_by_id_list);
766     RepatchBuffer repatchBuffer(codeBlock);
767     if (stubInfo.u.getByIdList.list->didSelfPatching()) {
768         repatchBuffer.relink(
769             stubInfo.callReturnLocation.jumpAtOffset(
770                 stubInfo.patch.deltaCallToJump),
771             CodeLocationLabel(stubRoutine->code().code()));
772         return;
773     }
774     
775     replaceWithJump(repatchBuffer, stubInfo, stubRoutine->code().code());
776 }
777
778 static InlineCacheAction tryBuildGetByIDList(ExecState* exec, JSValue baseValue, const Identifier& ident, const PropertySlot& slot, StructureStubInfo& stubInfo)
779 {
780     if (!baseValue.isCell()
781         || !slot.isCacheable())
782         return GiveUpOnCache;
783
784     JSCell* baseCell = baseValue.asCell();
785     bool loadTargetFromProxy = false;
786     if (baseCell->type() == PureForwardingProxyType) {
787         baseValue = jsCast<JSProxy*>(baseCell)->target();
788         baseCell = baseValue.asCell();
789         loadTargetFromProxy = true;
790     }
791
792     VM* vm = &exec->vm();
793     CodeBlock* codeBlock = exec->codeBlock();
794
795     InlineCacheAction action = actionForCell(*vm, baseCell);
796     if (action != AttemptToCache)
797         return action;
798
799     Structure* structure = baseCell->structure(*vm);
800     TypeInfo typeInfo = structure->typeInfo();
801
802     if (stubInfo.patch.spillMode == NeedToSpill) {
803         // We cannot do as much inline caching if the registers were not flushed prior to this GetById. In particular,
804         // non-Value cached properties require planting calls, which requires registers to have been flushed. Thus,
805         // if registers were not flushed, don't do non-Value caching.
806         if (!slot.isCacheableValue())
807             return GiveUpOnCache;
808     }
809     
810     PropertyOffset offset = slot.cachedOffset();
811     StructureChain* prototypeChain = 0;
812     size_t count = 0;
813     
814     if (slot.slotBase() != baseValue) {
815         if (typeInfo.prohibitsPropertyCaching() || structure->isDictionary())
816             return GiveUpOnCache;
817         
818         count = normalizePrototypeChainForChainAccess(
819             exec, baseValue, slot.slotBase(), ident, offset);
820         if (count == InvalidPrototypeChain)
821             return GiveUpOnCache;
822         prototypeChain = structure->prototypeChain(exec);
823     }
824     
825     PolymorphicGetByIdList* list = PolymorphicGetByIdList::from(stubInfo);
826     if (list->isFull()) {
827         // We need this extra check because of recursion.
828         return GiveUpOnCache;
829     }
830     
831     RefPtr<JITStubRoutine> stubRoutine;
832     generateByIdStub(
833         exec, kindFor(slot), ident, customFor(slot), stubInfo, prototypeChain, count, offset, 
834         structure, loadTargetFromProxy, slot.watchpointSet(), 
835         stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone),
836         CodeLocationLabel(list->currentSlowPathTarget(stubInfo)), stubRoutine);
837     
838     GetByIdAccess::AccessType accessType;
839     if (slot.isCacheableValue())
840         accessType = slot.watchpointSet() ? GetByIdAccess::WatchedStub : GetByIdAccess::SimpleStub;
841     else if (slot.isCacheableGetter())
842         accessType = GetByIdAccess::Getter;
843     else
844         accessType = GetByIdAccess::CustomGetter;
845     
846     list->addAccess(GetByIdAccess(
847         *vm, codeBlock->ownerExecutable(), accessType, stubRoutine, structure,
848         prototypeChain, count));
849     
850     patchJumpToGetByIdStub(codeBlock, stubInfo, stubRoutine.get());
851     
852     return list->isFull() ? GiveUpOnCache : RetryCacheLater;
853 }
854
855 void buildGetByIDList(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)
856 {
857     GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);
858     
859     if (tryBuildGetByIDList(exec, baseValue, propertyName, slot, stubInfo) == GiveUpOnCache)
860         repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, operationGetById);
861 }
862
863 static V_JITOperation_ESsiJJI appropriateGenericPutByIdFunction(const PutPropertySlot &slot, PutKind putKind)
864 {
865     if (slot.isStrictMode()) {
866         if (putKind == Direct)
867             return operationPutByIdDirectStrict;
868         return operationPutByIdStrict;
869     }
870     if (putKind == Direct)
871         return operationPutByIdDirectNonStrict;
872     return operationPutByIdNonStrict;
873 }
874
875 static V_JITOperation_ESsiJJI appropriateListBuildingPutByIdFunction(const PutPropertySlot &slot, PutKind putKind)
876 {
877     if (slot.isStrictMode()) {
878         if (putKind == Direct)
879             return operationPutByIdDirectStrictBuildList;
880         return operationPutByIdStrictBuildList;
881     }
882     if (putKind == Direct)
883         return operationPutByIdDirectNonStrictBuildList;
884     return operationPutByIdNonStrictBuildList;
885 }
886
887 static void emitPutReplaceStub(
888     ExecState* exec,
889     JSValue,
890     const Identifier&,
891     const PutPropertySlot& slot,
892     StructureStubInfo& stubInfo,
893     PutKind,
894     Structure* structure,
895     CodeLocationLabel failureLabel,
896     RefPtr<JITStubRoutine>& stubRoutine)
897 {
898     VM* vm = &exec->vm();
899     GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
900 #if USE(JSVALUE32_64)
901     GPRReg valueTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);
902 #endif
903     GPRReg valueGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);
904
905     ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
906     allocator.lock(baseGPR);
907 #if USE(JSVALUE32_64)
908     allocator.lock(valueTagGPR);
909 #endif
910     allocator.lock(valueGPR);
911     
912     GPRReg scratchGPR1 = allocator.allocateScratchGPR();
913
914     CCallHelpers stubJit(vm, exec->codeBlock());
915
916     allocator.preserveReusedRegistersByPushing(stubJit);
917
918     MacroAssembler::Jump badStructure = branchStructure(stubJit,
919         MacroAssembler::NotEqual,
920         MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()),
921         structure);
922
923 #if USE(JSVALUE64)
924     if (isInlineOffset(slot.cachedOffset()))
925         stubJit.store64(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue)));
926     else {
927         stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);
928         stubJit.store64(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue)));
929     }
930 #elif USE(JSVALUE32_64)
931     if (isInlineOffset(slot.cachedOffset())) {
932         stubJit.store32(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
933         stubJit.store32(valueTagGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
934     } else {
935         stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);
936         stubJit.store32(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
937         stubJit.store32(valueTagGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
938     }
939 #endif
940     
941     MacroAssembler::Jump success;
942     MacroAssembler::Jump failure;
943     
944     if (allocator.didReuseRegisters()) {
945         allocator.restoreReusedRegistersByPopping(stubJit);
946         success = stubJit.jump();
947         
948         badStructure.link(&stubJit);
949         allocator.restoreReusedRegistersByPopping(stubJit);
950         failure = stubJit.jump();
951     } else {
952         success = stubJit.jump();
953         failure = badStructure;
954     }
955     
956     LinkBuffer patchBuffer(*vm, &stubJit, exec->codeBlock());
957     patchBuffer.link(success, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));
958     patchBuffer.link(failure, failureLabel);
959             
960     stubRoutine = FINALIZE_CODE_FOR_STUB(
961         exec->codeBlock(), patchBuffer,
962         ("PutById replace stub for %s, return point %p",
963             toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(
964                 stubInfo.patch.deltaCallToDone).executableAddress()));
965 }
966
967 static void emitPutTransitionStub(
968     ExecState* exec,
969     JSValue,
970     const Identifier&,
971     const PutPropertySlot& slot,
972     StructureStubInfo& stubInfo,
973     PutKind putKind,
974     Structure* structure,
975     Structure* oldStructure,
976     StructureChain* prototypeChain,
977     CodeLocationLabel failureLabel,
978     RefPtr<JITStubRoutine>& stubRoutine)
979 {
980     VM* vm = &exec->vm();
981
982     GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
983 #if USE(JSVALUE32_64)
984     GPRReg valueTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);
985 #endif
986     GPRReg valueGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);
987     
988     ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
989     allocator.lock(baseGPR);
990 #if USE(JSVALUE32_64)
991     allocator.lock(valueTagGPR);
992 #endif
993     allocator.lock(valueGPR);
994     
995     CCallHelpers stubJit(vm);
996     
997     bool needThirdScratch = false;
998     if (structure->outOfLineCapacity() != oldStructure->outOfLineCapacity()
999         && oldStructure->outOfLineCapacity()) {
1000         needThirdScratch = true;
1001     }
1002
1003     GPRReg scratchGPR1 = allocator.allocateScratchGPR();
1004     ASSERT(scratchGPR1 != baseGPR);
1005     ASSERT(scratchGPR1 != valueGPR);
1006     
1007     GPRReg scratchGPR2 = allocator.allocateScratchGPR();
1008     ASSERT(scratchGPR2 != baseGPR);
1009     ASSERT(scratchGPR2 != valueGPR);
1010     ASSERT(scratchGPR2 != scratchGPR1);
1011
1012     GPRReg scratchGPR3;
1013     if (needThirdScratch) {
1014         scratchGPR3 = allocator.allocateScratchGPR();
1015         ASSERT(scratchGPR3 != baseGPR);
1016         ASSERT(scratchGPR3 != valueGPR);
1017         ASSERT(scratchGPR3 != scratchGPR1);
1018         ASSERT(scratchGPR3 != scratchGPR2);
1019     } else
1020         scratchGPR3 = InvalidGPRReg;
1021     
1022     allocator.preserveReusedRegistersByPushing(stubJit);
1023
1024     MacroAssembler::JumpList failureCases;
1025             
1026     ASSERT(oldStructure->transitionWatchpointSetHasBeenInvalidated());
1027     
1028     failureCases.append(branchStructure(stubJit,
1029         MacroAssembler::NotEqual, 
1030         MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), 
1031         oldStructure));
1032     
1033     addStructureTransitionCheck(
1034         oldStructure->storedPrototype(), exec->codeBlock(), stubInfo, stubJit, failureCases,
1035         scratchGPR1);
1036             
1037     if (putKind == NotDirect) {
1038         for (WriteBarrier<Structure>* it = prototypeChain->head(); *it; ++it) {
1039             addStructureTransitionCheck(
1040                 (*it)->storedPrototype(), exec->codeBlock(), stubInfo, stubJit, failureCases,
1041                 scratchGPR1);
1042         }
1043     }
1044
1045     MacroAssembler::JumpList slowPath;
1046     
1047     bool scratchGPR1HasStorage = false;
1048     
1049     if (structure->outOfLineCapacity() != oldStructure->outOfLineCapacity()) {
1050         size_t newSize = structure->outOfLineCapacity() * sizeof(JSValue);
1051         CopiedAllocator* copiedAllocator = &vm->heap.storageAllocator();
1052         
1053         if (!oldStructure->outOfLineCapacity()) {
1054             stubJit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR1);
1055             slowPath.append(stubJit.branchSubPtr(MacroAssembler::Signed, MacroAssembler::TrustedImm32(newSize), scratchGPR1));
1056             stubJit.storePtr(scratchGPR1, &copiedAllocator->m_currentRemaining);
1057             stubJit.negPtr(scratchGPR1);
1058             stubJit.addPtr(MacroAssembler::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR1);
1059             stubJit.addPtr(MacroAssembler::TrustedImm32(sizeof(JSValue)), scratchGPR1);
1060         } else {
1061             size_t oldSize = oldStructure->outOfLineCapacity() * sizeof(JSValue);
1062             ASSERT(newSize > oldSize);
1063             
1064             stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR3);
1065             stubJit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR1);
1066             slowPath.append(stubJit.branchSubPtr(MacroAssembler::Signed, MacroAssembler::TrustedImm32(newSize), scratchGPR1));
1067             stubJit.storePtr(scratchGPR1, &copiedAllocator->m_currentRemaining);
1068             stubJit.negPtr(scratchGPR1);
1069             stubJit.addPtr(MacroAssembler::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR1);
1070             stubJit.addPtr(MacroAssembler::TrustedImm32(sizeof(JSValue)), scratchGPR1);
1071             // We have scratchGPR1 = new storage, scratchGPR3 = old storage, scratchGPR2 = available
1072             for (size_t offset = 0; offset < oldSize; offset += sizeof(void*)) {
1073                 stubJit.loadPtr(MacroAssembler::Address(scratchGPR3, -static_cast<ptrdiff_t>(offset + sizeof(JSValue) + sizeof(void*))), scratchGPR2);
1074                 stubJit.storePtr(scratchGPR2, MacroAssembler::Address(scratchGPR1, -static_cast<ptrdiff_t>(offset + sizeof(JSValue) + sizeof(void*))));
1075             }
1076         }
1077         
1078         stubJit.storePtr(scratchGPR1, MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()));
1079         scratchGPR1HasStorage = true;
1080     }
1081
1082     ASSERT(oldStructure->typeInfo().type() == structure->typeInfo().type());
1083     ASSERT(oldStructure->typeInfo().inlineTypeFlags() == structure->typeInfo().inlineTypeFlags());
1084     ASSERT(oldStructure->indexingType() == structure->indexingType());
1085 #if USE(JSVALUE64)
1086     uint32_t val = structure->id();
1087 #else
1088     uint32_t val = reinterpret_cast<uint32_t>(structure->id());
1089 #endif
1090     stubJit.store32(MacroAssembler::TrustedImm32(val), MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()));
1091 #if USE(JSVALUE64)
1092     if (isInlineOffset(slot.cachedOffset()))
1093         stubJit.store64(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue)));
1094     else {
1095         if (!scratchGPR1HasStorage)
1096             stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);
1097         stubJit.store64(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue)));
1098     }
1099 #elif USE(JSVALUE32_64)
1100     if (isInlineOffset(slot.cachedOffset())) {
1101         stubJit.store32(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
1102         stubJit.store32(valueTagGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
1103     } else {
1104         if (!scratchGPR1HasStorage)
1105             stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);
1106         stubJit.store32(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
1107         stubJit.store32(valueTagGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
1108     }
1109 #endif
1110     
1111     MacroAssembler::Jump success;
1112     MacroAssembler::Jump failure;
1113             
1114     if (allocator.didReuseRegisters()) {
1115         allocator.restoreReusedRegistersByPopping(stubJit);
1116         success = stubJit.jump();
1117
1118         failureCases.link(&stubJit);
1119         allocator.restoreReusedRegistersByPopping(stubJit);
1120         failure = stubJit.jump();
1121     } else
1122         success = stubJit.jump();
1123     
1124     MacroAssembler::Call operationCall;
1125     MacroAssembler::Jump successInSlowPath;
1126     
1127     if (structure->outOfLineCapacity() != oldStructure->outOfLineCapacity()) {
1128         slowPath.link(&stubJit);
1129         
1130         allocator.restoreReusedRegistersByPopping(stubJit);
1131         ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(allocator.desiredScratchBufferSizeForCall());
1132         allocator.preserveUsedRegistersToScratchBufferForCall(stubJit, scratchBuffer, scratchGPR1);
1133 #if USE(JSVALUE64)
1134         stubJit.setupArgumentsWithExecState(baseGPR, MacroAssembler::TrustedImmPtr(structure), MacroAssembler::TrustedImm32(slot.cachedOffset()), valueGPR);
1135 #else
1136         stubJit.setupArgumentsWithExecState(baseGPR, MacroAssembler::TrustedImmPtr(structure), MacroAssembler::TrustedImm32(slot.cachedOffset()), valueGPR, valueTagGPR);
1137 #endif
1138         operationCall = stubJit.call();
1139         allocator.restoreUsedRegistersFromScratchBufferForCall(stubJit, scratchBuffer, scratchGPR1);
1140         successInSlowPath = stubJit.jump();
1141     }
1142     
1143     LinkBuffer patchBuffer(*vm, &stubJit, exec->codeBlock());
1144     patchBuffer.link(success, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));
1145     if (allocator.didReuseRegisters())
1146         patchBuffer.link(failure, failureLabel);
1147     else
1148         patchBuffer.link(failureCases, failureLabel);
1149     if (structure->outOfLineCapacity() != oldStructure->outOfLineCapacity()) {
1150         patchBuffer.link(operationCall, operationReallocateStorageAndFinishPut);
1151         patchBuffer.link(successInSlowPath, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));
1152     }
1153     
1154     stubRoutine =
1155         createJITStubRoutine(
1156             FINALIZE_CODE_FOR(
1157                 exec->codeBlock(), patchBuffer,
1158                 ("PutById %stransition stub (%p -> %p) for %s, return point %p",
1159                     structure->outOfLineCapacity() != oldStructure->outOfLineCapacity() ? "reallocating " : "",
1160                     oldStructure, structure,
1161                     toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(
1162                         stubInfo.patch.deltaCallToDone).executableAddress())),
1163             *vm,
1164             exec->codeBlock()->ownerExecutable(),
1165             structure->outOfLineCapacity() != oldStructure->outOfLineCapacity(),
1166             structure);
1167 }
1168
1169 static InlineCacheAction tryCachePutByID(ExecState* exec, JSValue baseValue, const Identifier& ident, const PutPropertySlot& slot, StructureStubInfo& stubInfo, PutKind putKind)
1170 {
1171     if (Options::forceICFailure())
1172         return GiveUpOnCache;
1173     
1174     CodeBlock* codeBlock = exec->codeBlock();
1175     VM* vm = &exec->vm();
1176
1177     if (!baseValue.isCell())
1178         return GiveUpOnCache;
1179     JSCell* baseCell = baseValue.asCell();
1180     Structure* structure = baseCell->structure();
1181     Structure* oldStructure = structure->previousID();
1182     
1183     if (!slot.isCacheablePut() && !slot.isCacheableCustom() && !slot.isCacheableSetter())
1184         return GiveUpOnCache;
1185     if (!structure->propertyAccessesAreCacheable())
1186         return GiveUpOnCache;
1187
1188     // Optimize self access.
1189     if (slot.base() == baseValue && slot.isCacheablePut()) {
1190         if (slot.type() == PutPropertySlot::NewProperty) {
1191             if (structure->isDictionary())
1192                 return GiveUpOnCache;
1193             
1194             // Skip optimizing the case where we need a realloc, if we don't have
1195             // enough registers to make it happen.
1196             if (GPRInfo::numberOfRegisters < 6
1197                 && oldStructure->outOfLineCapacity() != structure->outOfLineCapacity()
1198                 && oldStructure->outOfLineCapacity())
1199                 return GiveUpOnCache;
1200             
1201             // Skip optimizing the case where we need realloc, and the structure has
1202             // indexing storage.
1203             // FIXME: We shouldn't skip this!  Implement it!
1204             // https://bugs.webkit.org/show_bug.cgi?id=130914
1205             if (oldStructure->couldHaveIndexingHeader())
1206                 return GiveUpOnCache;
1207             
1208             if (normalizePrototypeChain(exec, baseCell) == InvalidPrototypeChain)
1209                 return GiveUpOnCache;
1210             
1211             StructureChain* prototypeChain = structure->prototypeChain(exec);
1212             
1213             emitPutTransitionStub(
1214                 exec, baseValue, ident, slot, stubInfo, putKind,
1215                 structure, oldStructure, prototypeChain,
1216                 stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase),
1217                 stubInfo.stubRoutine);
1218             
1219             RepatchBuffer repatchBuffer(codeBlock);
1220             repatchBuffer.relink(
1221                 stubInfo.callReturnLocation.jumpAtOffset(
1222                     stubInfo.patch.deltaCallToJump),
1223                 CodeLocationLabel(stubInfo.stubRoutine->code().code()));
1224             repatchCall(repatchBuffer, stubInfo.callReturnLocation, appropriateListBuildingPutByIdFunction(slot, putKind));
1225             
1226             stubInfo.initPutByIdTransition(*vm, codeBlock->ownerExecutable(), oldStructure, structure, prototypeChain, putKind == Direct);
1227             
1228             return RetryCacheLater;
1229         }
1230
1231         if (!MacroAssembler::isPtrAlignedAddressOffset(offsetRelativeToPatchedStorage(slot.cachedOffset())))
1232             return GiveUpOnCache;
1233
1234         repatchByIdSelfAccess(*vm, codeBlock, stubInfo, structure, ident, slot.cachedOffset(), appropriateListBuildingPutByIdFunction(slot, putKind), false);
1235         stubInfo.initPutByIdReplace(*vm, codeBlock->ownerExecutable(), structure);
1236         return RetryCacheLater;
1237     }
1238     if ((slot.isCacheableCustom() || slot.isCacheableSetter())
1239         && stubInfo.patch.spillMode == DontSpill) {
1240         RefPtr<JITStubRoutine> stubRoutine;
1241
1242         StructureChain* prototypeChain = 0;
1243         PropertyOffset offset = slot.cachedOffset();
1244         size_t count = 0;
1245         if (baseValue != slot.base()) {
1246             count = normalizePrototypeChainForChainAccess(exec, baseCell, slot.base(), ident, offset);
1247             if (count == InvalidPrototypeChain)
1248                 return GiveUpOnCache;
1249
1250             prototypeChain = structure->prototypeChain(exec);
1251         }
1252         PolymorphicPutByIdList* list;
1253         list = PolymorphicPutByIdList::from(putKind, stubInfo);
1254
1255         generateByIdStub(
1256             exec, kindFor(slot), ident, customFor(slot), stubInfo, prototypeChain, count,
1257             offset, structure, false, nullptr,
1258             stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone),
1259             stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase),
1260             stubRoutine);
1261
1262         list->addAccess(PutByIdAccess::setter(
1263             *vm, codeBlock->ownerExecutable(),
1264             slot.isCacheableSetter() ? PutByIdAccess::Setter : PutByIdAccess::CustomSetter,
1265             structure, prototypeChain, slot.customSetter(), stubRoutine));
1266
1267         RepatchBuffer repatchBuffer(codeBlock);
1268         repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), CodeLocationLabel(stubRoutine->code().code()));
1269         repatchCall(repatchBuffer, stubInfo.callReturnLocation, appropriateListBuildingPutByIdFunction(slot, putKind));
1270         RELEASE_ASSERT(!list->isFull());
1271         return RetryCacheLater;
1272     }
1273
1274     return GiveUpOnCache;
1275 }
1276
1277 void repatchPutByID(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PutPropertySlot& slot, StructureStubInfo& stubInfo, PutKind putKind)
1278 {
1279     GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);
1280     
1281     if (tryCachePutByID(exec, baseValue, propertyName, slot, stubInfo, putKind) == GiveUpOnCache)
1282         repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, appropriateGenericPutByIdFunction(slot, putKind));
1283 }
1284
1285 static InlineCacheAction tryBuildPutByIdList(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PutPropertySlot& slot, StructureStubInfo& stubInfo, PutKind putKind)
1286 {
1287     CodeBlock* codeBlock = exec->codeBlock();
1288     VM* vm = &exec->vm();
1289
1290     if (!baseValue.isCell())
1291         return GiveUpOnCache;
1292     JSCell* baseCell = baseValue.asCell();
1293     Structure* structure = baseCell->structure();
1294     Structure* oldStructure = structure->previousID();
1295     
1296     
1297     if (!slot.isCacheablePut() && !slot.isCacheableCustom() && !slot.isCacheableSetter())
1298         return GiveUpOnCache;
1299
1300     if (!structure->propertyAccessesAreCacheable())
1301         return GiveUpOnCache;
1302
1303     // Optimize self access.
1304     if (slot.base() == baseValue && slot.isCacheablePut()) {
1305         PolymorphicPutByIdList* list;
1306         RefPtr<JITStubRoutine> stubRoutine;
1307         
1308         if (slot.type() == PutPropertySlot::NewProperty) {
1309             if (structure->isDictionary())
1310                 return GiveUpOnCache;
1311             
1312             // Skip optimizing the case where we need a realloc, if we don't have
1313             // enough registers to make it happen.
1314             if (GPRInfo::numberOfRegisters < 6
1315                 && oldStructure->outOfLineCapacity() != structure->outOfLineCapacity()
1316                 && oldStructure->outOfLineCapacity())
1317                 return GiveUpOnCache;
1318             
1319             // Skip optimizing the case where we need realloc, and the structure has
1320             // indexing storage.
1321             if (oldStructure->couldHaveIndexingHeader())
1322                 return GiveUpOnCache;
1323             
1324             if (normalizePrototypeChain(exec, baseCell) == InvalidPrototypeChain)
1325                 return GiveUpOnCache;
1326             
1327             StructureChain* prototypeChain = structure->prototypeChain(exec);
1328             
1329             list = PolymorphicPutByIdList::from(putKind, stubInfo);
1330             if (list->isFull())
1331                 return GiveUpOnCache; // Will get here due to recursion.
1332             
1333             // We're now committed to creating the stub. Mogrify the meta-data accordingly.
1334             emitPutTransitionStub(
1335                 exec, baseValue, propertyName, slot, stubInfo, putKind,
1336                 structure, oldStructure, prototypeChain,
1337                 CodeLocationLabel(list->currentSlowPathTarget()),
1338                 stubRoutine);
1339             
1340             list->addAccess(
1341                 PutByIdAccess::transition(
1342                     *vm, codeBlock->ownerExecutable(),
1343                     oldStructure, structure, prototypeChain,
1344                     stubRoutine));
1345         } else {
1346             list = PolymorphicPutByIdList::from(putKind, stubInfo);
1347             if (list->isFull())
1348                 return GiveUpOnCache; // Will get here due to recursion.
1349             
1350             // We're now committed to creating the stub. Mogrify the meta-data accordingly.
1351             emitPutReplaceStub(
1352                 exec, baseValue, propertyName, slot, stubInfo, putKind,
1353                 structure, CodeLocationLabel(list->currentSlowPathTarget()), stubRoutine);
1354             
1355             list->addAccess(
1356                 PutByIdAccess::replace(
1357                     *vm, codeBlock->ownerExecutable(),
1358                     structure, stubRoutine));
1359         }
1360         
1361         RepatchBuffer repatchBuffer(codeBlock);
1362         repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), CodeLocationLabel(stubRoutine->code().code()));
1363         
1364         if (list->isFull())
1365             repatchCall(repatchBuffer, stubInfo.callReturnLocation, appropriateGenericPutByIdFunction(slot, putKind));
1366         
1367         return RetryCacheLater;
1368     }
1369
1370     if ((slot.isCacheableCustom() || slot.isCacheableSetter())
1371         && stubInfo.patch.spillMode == DontSpill) {
1372         RefPtr<JITStubRoutine> stubRoutine;
1373         StructureChain* prototypeChain = 0;
1374         PropertyOffset offset = slot.cachedOffset();
1375         size_t count = 0;
1376         if (baseValue != slot.base()) {
1377             count = normalizePrototypeChainForChainAccess(exec, baseCell, slot.base(), propertyName, offset);
1378             if (count == InvalidPrototypeChain)
1379                 return GiveUpOnCache;
1380
1381             prototypeChain = structure->prototypeChain(exec);
1382         }
1383         PolymorphicPutByIdList* list;
1384         list = PolymorphicPutByIdList::from(putKind, stubInfo);
1385
1386         generateByIdStub(
1387             exec, kindFor(slot), propertyName, customFor(slot), stubInfo, prototypeChain, count,
1388             offset, structure, false, nullptr,
1389             stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone),
1390             CodeLocationLabel(list->currentSlowPathTarget()),
1391             stubRoutine);
1392
1393         list->addAccess(PutByIdAccess::setter(
1394             *vm, codeBlock->ownerExecutable(),
1395             slot.isCacheableSetter() ? PutByIdAccess::Setter : PutByIdAccess::CustomSetter,
1396             structure, prototypeChain, slot.customSetter(), stubRoutine));
1397
1398         RepatchBuffer repatchBuffer(codeBlock);
1399         repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), CodeLocationLabel(stubRoutine->code().code()));
1400         if (list->isFull())
1401             repatchCall(repatchBuffer, stubInfo.callReturnLocation, appropriateGenericPutByIdFunction(slot, putKind));
1402
1403         return RetryCacheLater;
1404     }
1405     return GiveUpOnCache;
1406 }
1407
1408 void buildPutByIdList(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PutPropertySlot& slot, StructureStubInfo& stubInfo, PutKind putKind)
1409 {
1410     GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);
1411     
1412     if (tryBuildPutByIdList(exec, baseValue, propertyName, slot, stubInfo, putKind) == GiveUpOnCache)
1413         repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, appropriateGenericPutByIdFunction(slot, putKind));
1414 }
1415
1416 static InlineCacheAction tryRepatchIn(
1417     ExecState* exec, JSCell* base, const Identifier& ident, bool wasFound,
1418     const PropertySlot& slot, StructureStubInfo& stubInfo)
1419 {
1420     if (Options::forceICFailure())
1421         return GiveUpOnCache;
1422     
1423     if (!base->structure()->propertyAccessesAreCacheable())
1424         return GiveUpOnCache;
1425     
1426     if (wasFound) {
1427         if (!slot.isCacheable())
1428             return GiveUpOnCache;
1429     }
1430     
1431     CodeBlock* codeBlock = exec->codeBlock();
1432     VM* vm = &exec->vm();
1433     Structure* structure = base->structure();
1434     
1435     PropertyOffset offsetIgnored;
1436     size_t count = normalizePrototypeChainForChainAccess(exec, base, wasFound ? slot.slotBase() : JSValue(), ident, offsetIgnored);
1437     if (count == InvalidPrototypeChain)
1438         return GiveUpOnCache;
1439     
1440     PolymorphicAccessStructureList* polymorphicStructureList;
1441     int listIndex;
1442     
1443     CodeLocationLabel successLabel = stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone);
1444     CodeLocationLabel slowCaseLabel;
1445     
1446     if (stubInfo.accessType == access_unset) {
1447         polymorphicStructureList = new PolymorphicAccessStructureList();
1448         stubInfo.initInList(polymorphicStructureList, 0);
1449         slowCaseLabel = stubInfo.callReturnLocation.labelAtOffset(
1450             stubInfo.patch.deltaCallToSlowCase);
1451         listIndex = 0;
1452     } else {
1453         RELEASE_ASSERT(stubInfo.accessType == access_in_list);
1454         polymorphicStructureList = stubInfo.u.inList.structureList;
1455         listIndex = stubInfo.u.inList.listSize;
1456         slowCaseLabel = CodeLocationLabel(polymorphicStructureList->list[listIndex - 1].stubRoutine->code().code());
1457         
1458         if (listIndex == POLYMORPHIC_LIST_CACHE_SIZE)
1459             return GiveUpOnCache;
1460     }
1461     
1462     StructureChain* chain = structure->prototypeChain(exec);
1463     RefPtr<JITStubRoutine> stubRoutine;
1464     
1465     {
1466         GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
1467         GPRReg resultGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);
1468         GPRReg scratchGPR = TempRegisterSet(stubInfo.patch.usedRegisters).getFreeGPR();
1469         
1470         CCallHelpers stubJit(vm);
1471         
1472         bool needToRestoreScratch;
1473         if (scratchGPR == InvalidGPRReg) {
1474             scratchGPR = AssemblyHelpers::selectScratchGPR(baseGPR, resultGPR);
1475             stubJit.pushToSave(scratchGPR);
1476             needToRestoreScratch = true;
1477         } else
1478             needToRestoreScratch = false;
1479         
1480         MacroAssembler::JumpList failureCases;
1481         failureCases.append(branchStructure(stubJit,
1482             MacroAssembler::NotEqual,
1483             MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()),
1484             structure));
1485
1486         CodeBlock* codeBlock = exec->codeBlock();
1487         if (structure->typeInfo().newImpurePropertyFiresWatchpoints())
1488             vm->registerWatchpointForImpureProperty(ident, stubInfo.addWatchpoint(codeBlock));
1489
1490         if (slot.watchpointSet())
1491             slot.watchpointSet()->add(stubInfo.addWatchpoint(codeBlock));
1492
1493         Structure* currStructure = structure;
1494         WriteBarrier<Structure>* it = chain->head();
1495         for (unsigned i = 0; i < count; ++i, ++it) {
1496             JSObject* prototype = asObject(currStructure->prototypeForLookup(exec));
1497             Structure* protoStructure = prototype->structure();
1498             addStructureTransitionCheck(
1499                 prototype, protoStructure, exec->codeBlock(), stubInfo, stubJit,
1500                 failureCases, scratchGPR);
1501             if (protoStructure->typeInfo().newImpurePropertyFiresWatchpoints())
1502                 vm->registerWatchpointForImpureProperty(ident, stubInfo.addWatchpoint(codeBlock));
1503             currStructure = it->get();
1504         }
1505         
1506 #if USE(JSVALUE64)
1507         stubJit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(wasFound))), resultGPR);
1508 #else
1509         stubJit.move(MacroAssembler::TrustedImm32(wasFound), resultGPR);
1510 #endif
1511         
1512         MacroAssembler::Jump success, fail;
1513         
1514         emitRestoreScratch(stubJit, needToRestoreScratch, scratchGPR, success, fail, failureCases);
1515         
1516         LinkBuffer patchBuffer(*vm, &stubJit, exec->codeBlock());
1517
1518         linkRestoreScratch(patchBuffer, needToRestoreScratch, success, fail, failureCases, successLabel, slowCaseLabel);
1519         
1520         stubRoutine = FINALIZE_CODE_FOR_STUB(
1521             exec->codeBlock(), patchBuffer,
1522             ("In (found = %s) stub for %s, return point %p",
1523                 wasFound ? "yes" : "no", toCString(*exec->codeBlock()).data(),
1524                 successLabel.executableAddress()));
1525     }
1526     
1527     polymorphicStructureList->list[listIndex].set(*vm, codeBlock->ownerExecutable(), stubRoutine, structure, true);
1528     stubInfo.u.inList.listSize++;
1529     
1530     RepatchBuffer repatchBuffer(codeBlock);
1531     repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), CodeLocationLabel(stubRoutine->code().code()));
1532     
1533     return listIndex < (POLYMORPHIC_LIST_CACHE_SIZE - 1) ? RetryCacheLater : GiveUpOnCache;
1534 }
1535
1536 void repatchIn(
1537     ExecState* exec, JSCell* base, const Identifier& ident, bool wasFound,
1538     const PropertySlot& slot, StructureStubInfo& stubInfo)
1539 {
1540     if (tryRepatchIn(exec, base, ident, wasFound, slot, stubInfo) == GiveUpOnCache)
1541         repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, operationIn);
1542 }
1543
1544 static void linkSlowFor(
1545     RepatchBuffer& repatchBuffer, VM* vm, CallLinkInfo& callLinkInfo,
1546     CodeSpecializationKind kind, RegisterPreservationMode registers)
1547 {
1548     repatchBuffer.relink(
1549         callLinkInfo.callReturnLocation,
1550         vm->getCTIStub(virtualThunkGeneratorFor(kind, registers)).code());
1551 }
1552
1553 void linkFor(
1554     ExecState* exec, CallLinkInfo& callLinkInfo, CodeBlock* calleeCodeBlock,
1555     JSFunction* callee, MacroAssemblerCodePtr codePtr, CodeSpecializationKind kind,
1556     RegisterPreservationMode registers)
1557 {
1558     ASSERT(!callLinkInfo.stub);
1559     
1560     CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
1561
1562     // If you're being call-linked from a DFG caller then you obviously didn't get inlined.
1563     if (calleeCodeBlock && JITCode::isOptimizingJIT(callerCodeBlock->jitType()))
1564         calleeCodeBlock->m_shouldAlwaysBeInlined = false;
1565     
1566     VM* vm = callerCodeBlock->vm();
1567     
1568     RepatchBuffer repatchBuffer(callerCodeBlock);
1569     
1570     ASSERT(!callLinkInfo.isLinked());
1571     callLinkInfo.callee.set(exec->callerFrame()->vm(), callLinkInfo.hotPathBegin, callerCodeBlock->ownerExecutable(), callee);
1572     callLinkInfo.lastSeenCallee.set(exec->callerFrame()->vm(), callerCodeBlock->ownerExecutable(), callee);
1573     if (shouldShowDisassemblyFor(callerCodeBlock))
1574         dataLog("Linking call in ", *callerCodeBlock, " at ", callLinkInfo.codeOrigin, " to ", pointerDump(calleeCodeBlock), ", entrypoint at ", codePtr, "\n");
1575     repatchBuffer.relink(callLinkInfo.hotPathOther, codePtr);
1576     
1577     if (calleeCodeBlock)
1578         calleeCodeBlock->linkIncomingCall(exec->callerFrame(), &callLinkInfo);
1579     
1580     if (kind == CodeForCall) {
1581         repatchBuffer.relink(callLinkInfo.callReturnLocation, vm->getCTIStub(linkClosureCallThunkGeneratorFor(registers)).code());
1582         return;
1583     }
1584     
1585     ASSERT(kind == CodeForConstruct);
1586     linkSlowFor(repatchBuffer, vm, callLinkInfo, CodeForConstruct, registers);
1587 }
1588
1589 void linkSlowFor(
1590     ExecState* exec, CallLinkInfo& callLinkInfo, CodeSpecializationKind kind,
1591     RegisterPreservationMode registers)
1592 {
1593     CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
1594     VM* vm = callerCodeBlock->vm();
1595     
1596     RepatchBuffer repatchBuffer(callerCodeBlock);
1597     
1598     linkSlowFor(repatchBuffer, vm, callLinkInfo, kind, registers);
1599 }
1600
1601 void linkClosureCall(
1602     ExecState* exec, CallLinkInfo& callLinkInfo, CodeBlock* calleeCodeBlock,
1603     Structure* structure, ExecutableBase* executable, MacroAssemblerCodePtr codePtr,
1604     RegisterPreservationMode registers)
1605 {
1606     ASSERT(!callLinkInfo.stub);
1607     
1608     CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
1609     VM* vm = callerCodeBlock->vm();
1610     
1611     GPRReg calleeGPR = static_cast<GPRReg>(callLinkInfo.calleeGPR);
1612     
1613     CCallHelpers stubJit(vm, callerCodeBlock);
1614     
1615     CCallHelpers::JumpList slowPath;
1616     
1617     ptrdiff_t offsetToFrame = -sizeof(CallerFrameAndPC);
1618
1619     if (!ASSERT_DISABLED) {
1620         CCallHelpers::Jump okArgumentCount = stubJit.branch32(
1621             CCallHelpers::Below, CCallHelpers::Address(CCallHelpers::stackPointerRegister, static_cast<ptrdiff_t>(sizeof(Register) * JSStack::ArgumentCount) + offsetToFrame + PayloadOffset), CCallHelpers::TrustedImm32(10000000));
1622         stubJit.abortWithReason(RepatchInsaneArgumentCount);
1623         okArgumentCount.link(&stubJit);
1624     }
1625
1626 #if USE(JSVALUE64)
1627     // We can safely clobber everything except the calleeGPR. We can't rely on tagMaskRegister
1628     // being set. So we do this the hard way.
1629     GPRReg scratch = AssemblyHelpers::selectScratchGPR(calleeGPR);
1630     stubJit.move(MacroAssembler::TrustedImm64(TagMask), scratch);
1631     slowPath.append(stubJit.branchTest64(CCallHelpers::NonZero, calleeGPR, scratch));
1632 #else
1633     // We would have already checked that the callee is a cell.
1634 #endif
1635     
1636     slowPath.append(
1637         branchStructure(stubJit,
1638             CCallHelpers::NotEqual,
1639             CCallHelpers::Address(calleeGPR, JSCell::structureIDOffset()),
1640             structure));
1641     
1642     slowPath.append(
1643         stubJit.branchPtr(
1644             CCallHelpers::NotEqual,
1645             CCallHelpers::Address(calleeGPR, JSFunction::offsetOfExecutable()),
1646             CCallHelpers::TrustedImmPtr(executable)));
1647     
1648     stubJit.loadPtr(
1649         CCallHelpers::Address(calleeGPR, JSFunction::offsetOfScopeChain()),
1650         GPRInfo::returnValueGPR);
1651     
1652 #if USE(JSVALUE64)
1653     stubJit.store64(
1654         GPRInfo::returnValueGPR,
1655         CCallHelpers::Address(MacroAssembler::stackPointerRegister, static_cast<ptrdiff_t>(sizeof(Register) * JSStack::ScopeChain) + offsetToFrame));
1656 #else
1657     stubJit.storePtr(
1658         GPRInfo::returnValueGPR,
1659         CCallHelpers::Address(MacroAssembler::stackPointerRegister, static_cast<ptrdiff_t>(sizeof(Register) * JSStack::ScopeChain) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload) + offsetToFrame));
1660     stubJit.store32(
1661         CCallHelpers::TrustedImm32(JSValue::CellTag),
1662         CCallHelpers::Address(MacroAssembler::stackPointerRegister, static_cast<ptrdiff_t>(sizeof(Register) * JSStack::ScopeChain) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag) + offsetToFrame));
1663 #endif
1664     
1665     AssemblyHelpers::Call call = stubJit.nearCall();
1666     AssemblyHelpers::Jump done = stubJit.jump();
1667     
1668     slowPath.link(&stubJit);
1669     stubJit.move(calleeGPR, GPRInfo::regT0);
1670 #if USE(JSVALUE32_64)
1671     stubJit.move(CCallHelpers::TrustedImm32(JSValue::CellTag), GPRInfo::regT1);
1672 #endif
1673     stubJit.move(CCallHelpers::TrustedImmPtr(&callLinkInfo), GPRInfo::regT2);
1674     stubJit.move(CCallHelpers::TrustedImmPtr(callLinkInfo.callReturnLocation.executableAddress()), GPRInfo::regT4);
1675     
1676     stubJit.restoreReturnAddressBeforeReturn(GPRInfo::regT4);
1677     AssemblyHelpers::Jump slow = stubJit.jump();
1678     
1679     LinkBuffer patchBuffer(*vm, &stubJit, callerCodeBlock);
1680     
1681     patchBuffer.link(call, FunctionPtr(codePtr.executableAddress()));
1682     if (JITCode::isOptimizingJIT(callerCodeBlock->jitType()))
1683         patchBuffer.link(done, callLinkInfo.callReturnLocation.labelAtOffset(0));
1684     else
1685         patchBuffer.link(done, callLinkInfo.hotPathOther.labelAtOffset(0));
1686     patchBuffer.link(slow, CodeLocationLabel(vm->getCTIStub(virtualThunkGeneratorFor(CodeForCall, registers)).code()));
1687     
1688     RefPtr<ClosureCallStubRoutine> stubRoutine = adoptRef(new ClosureCallStubRoutine(
1689         FINALIZE_CODE_FOR(
1690             callerCodeBlock, patchBuffer,
1691             ("Closure call stub for %s, return point %p, target %p (%s)",
1692                 toCString(*callerCodeBlock).data(), callLinkInfo.callReturnLocation.labelAtOffset(0).executableAddress(),
1693                 codePtr.executableAddress(), toCString(pointerDump(calleeCodeBlock)).data())),
1694         *vm, callerCodeBlock->ownerExecutable(), structure, executable, callLinkInfo.codeOrigin));
1695     
1696     RepatchBuffer repatchBuffer(callerCodeBlock);
1697     
1698     repatchBuffer.replaceWithJump(
1699         RepatchBuffer::startOfBranchPtrWithPatchOnRegister(callLinkInfo.hotPathBegin),
1700         CodeLocationLabel(stubRoutine->code().code()));
1701     linkSlowFor(repatchBuffer, vm, callLinkInfo, CodeForCall, registers);
1702     
1703     callLinkInfo.stub = stubRoutine.release();
1704     
1705     ASSERT(!calleeCodeBlock || calleeCodeBlock->isIncomingCallAlreadyLinked(&callLinkInfo));
1706 }
1707
1708 void resetGetByID(RepatchBuffer& repatchBuffer, StructureStubInfo& stubInfo)
1709 {
1710     repatchCall(repatchBuffer, stubInfo.callReturnLocation, operationGetByIdOptimize);
1711     CodeLocationDataLabel32 structureLabel = stubInfo.callReturnLocation.dataLabel32AtOffset(-(intptr_t)stubInfo.patch.deltaCheckImmToCall);
1712     if (MacroAssembler::canJumpReplacePatchableBranch32WithPatch()) {
1713         repatchBuffer.revertJumpReplacementToPatchableBranch32WithPatch(
1714             RepatchBuffer::startOfPatchableBranch32WithPatchOnAddress(structureLabel),
1715             MacroAssembler::Address(
1716                 static_cast<MacroAssembler::RegisterID>(stubInfo.patch.baseGPR),
1717                 JSCell::structureIDOffset()),
1718             static_cast<int32_t>(unusedPointer));
1719     }
1720     repatchBuffer.repatch(structureLabel, static_cast<int32_t>(unusedPointer));
1721 #if USE(JSVALUE64)
1722     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToLoadOrStore), 0);
1723 #else
1724     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), 0);
1725     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), 0);
1726 #endif
1727     repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
1728 }
1729
1730 void resetPutByID(RepatchBuffer& repatchBuffer, StructureStubInfo& stubInfo)
1731 {
1732     V_JITOperation_ESsiJJI unoptimizedFunction = bitwise_cast<V_JITOperation_ESsiJJI>(readCallTarget(repatchBuffer, stubInfo.callReturnLocation).executableAddress());
1733     V_JITOperation_ESsiJJI optimizedFunction;
1734     if (unoptimizedFunction == operationPutByIdStrict || unoptimizedFunction == operationPutByIdStrictBuildList)
1735         optimizedFunction = operationPutByIdStrictOptimize;
1736     else if (unoptimizedFunction == operationPutByIdNonStrict || unoptimizedFunction == operationPutByIdNonStrictBuildList)
1737         optimizedFunction = operationPutByIdNonStrictOptimize;
1738     else if (unoptimizedFunction == operationPutByIdDirectStrict || unoptimizedFunction == operationPutByIdDirectStrictBuildList)
1739         optimizedFunction = operationPutByIdDirectStrictOptimize;
1740     else {
1741         ASSERT(unoptimizedFunction == operationPutByIdDirectNonStrict || unoptimizedFunction == operationPutByIdDirectNonStrictBuildList);
1742         optimizedFunction = operationPutByIdDirectNonStrictOptimize;
1743     }
1744     repatchCall(repatchBuffer, stubInfo.callReturnLocation, optimizedFunction);
1745     CodeLocationDataLabel32 structureLabel = stubInfo.callReturnLocation.dataLabel32AtOffset(-(intptr_t)stubInfo.patch.deltaCheckImmToCall);
1746     if (MacroAssembler::canJumpReplacePatchableBranch32WithPatch()) {
1747         repatchBuffer.revertJumpReplacementToPatchableBranch32WithPatch(
1748             RepatchBuffer::startOfPatchableBranch32WithPatchOnAddress(structureLabel),
1749             MacroAssembler::Address(
1750                 static_cast<MacroAssembler::RegisterID>(stubInfo.patch.baseGPR),
1751                 JSCell::structureIDOffset()),
1752             static_cast<int32_t>(unusedPointer));
1753     }
1754     repatchBuffer.repatch(structureLabel, static_cast<int32_t>(unusedPointer));
1755 #if USE(JSVALUE64)
1756     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToLoadOrStore), 0);
1757 #else
1758     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), 0);
1759     repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), 0);
1760 #endif
1761     repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
1762 }
1763
1764 void resetIn(RepatchBuffer& repatchBuffer, StructureStubInfo& stubInfo)
1765 {
1766     repatchBuffer.relink(stubInfo.callReturnLocation.jumpAtOffset(stubInfo.patch.deltaCallToJump), stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));
1767 }
1768
1769 } // namespace JSC
1770
1771 #endif