e0230bbf51324847282b0f708171e4ee2ed86412
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLOSRExitCompiler.cpp
1 /*
2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLOSRExitCompiler.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "DFGOSRExitCompilerCommon.h"
32 #include "DFGOSRExitPreparation.h"
33 #include "FTLExitArgumentForOperand.h"
34 #include "FTLJITCode.h"
35 #include "FTLLocation.h"
36 #include "FTLOSRExit.h"
37 #include "FTLOperations.h"
38 #include "FTLState.h"
39 #include "FTLSaveRestore.h"
40 #include "LinkBuffer.h"
41 #include "MaxFrameExtentForSlowPathCall.h"
42 #include "OperandsInlines.h"
43 #include "JSCInlines.h"
44
45 namespace JSC { namespace FTL {
46
47 using namespace DFG;
48
49 static void reboxAccordingToFormat(
50     DataFormat format, AssemblyHelpers& jit, GPRReg value, GPRReg scratch1, GPRReg scratch2)
51 {
52     switch (format) {
53     case DataFormatInt32: {
54         jit.zeroExtend32ToPtr(value, value);
55         jit.or64(GPRInfo::tagTypeNumberRegister, value);
56         break;
57     }
58
59     case DataFormatInt52: {
60         jit.rshift64(AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), value);
61         jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch2);
62         jit.boxInt52(value, value, scratch1, FPRInfo::fpRegT0);
63         jit.move64ToDouble(scratch2, FPRInfo::fpRegT0);
64         break;
65     }
66
67     case DataFormatStrictInt52: {
68         jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch2);
69         jit.boxInt52(value, value, scratch1, FPRInfo::fpRegT0);
70         jit.move64ToDouble(scratch2, FPRInfo::fpRegT0);
71         break;
72     }
73
74     case DataFormatBoolean: {
75         jit.zeroExtend32ToPtr(value, value);
76         jit.or32(MacroAssembler::TrustedImm32(ValueFalse), value);
77         break;
78     }
79
80     case DataFormatJS: {
81         // Done already!
82         break;
83     }
84
85     case DataFormatDouble: {
86         jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch1);
87         jit.move64ToDouble(value, FPRInfo::fpRegT0);
88         jit.purifyNaN(FPRInfo::fpRegT0);
89         jit.boxDouble(FPRInfo::fpRegT0, value);
90         jit.move64ToDouble(scratch1, FPRInfo::fpRegT0);
91         break;
92     }
93
94     default:
95         RELEASE_ASSERT_NOT_REACHED();
96         break;
97     }
98 }
99
100 static void compileRecovery(
101     CCallHelpers& jit, const ExitValue& value,
102     Vector<B3::ValueRep>& valueReps,
103     char* registerScratch,
104     const HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*>& materializationToPointer)
105 {
106     switch (value.kind()) {
107     case ExitValueDead:
108         jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
109         break;
110             
111     case ExitValueConstant:
112         jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
113         break;
114             
115     case ExitValueArgument:
116         Location::forValueRep(valueReps[value.exitArgument().argument()]).restoreInto(
117             jit, registerScratch, GPRInfo::regT0);
118         break;
119             
120     case ExitValueInJSStack:
121     case ExitValueInJSStackAsInt32:
122     case ExitValueInJSStackAsInt52:
123     case ExitValueInJSStackAsDouble:
124         jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
125         break;
126             
127     case ExitValueRecovery:
128         Location::forValueRep(valueReps[value.rightRecoveryArgument()]).restoreInto(
129             jit, registerScratch, GPRInfo::regT1);
130         Location::forValueRep(valueReps[value.leftRecoveryArgument()]).restoreInto(
131             jit, registerScratch, GPRInfo::regT0);
132         switch (value.recoveryOpcode()) {
133         case AddRecovery:
134             switch (value.recoveryFormat()) {
135             case DataFormatInt32:
136                 jit.add32(GPRInfo::regT1, GPRInfo::regT0);
137                 break;
138             case DataFormatInt52:
139                 jit.add64(GPRInfo::regT1, GPRInfo::regT0);
140                 break;
141             default:
142                 RELEASE_ASSERT_NOT_REACHED();
143                 break;
144             }
145             break;
146         case SubRecovery:
147             switch (value.recoveryFormat()) {
148             case DataFormatInt32:
149                 jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
150                 break;
151             case DataFormatInt52:
152                 jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
153                 break;
154             default:
155                 RELEASE_ASSERT_NOT_REACHED();
156                 break;
157             }
158             break;
159         default:
160             RELEASE_ASSERT_NOT_REACHED();
161             break;
162         }
163         break;
164         
165     case ExitValueMaterializeNewObject:
166         jit.loadPtr(materializationToPointer.get(value.objectMaterialization()), GPRInfo::regT0);
167         break;
168             
169     default:
170         RELEASE_ASSERT_NOT_REACHED();
171         break;
172     }
173         
174     reboxAccordingToFormat(
175         value.dataFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
176 }
177
178 static void compileStub(
179     unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
180 {
181     // This code requires framePointerRegister is the same as callFrameRegister
182     static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
183
184     CCallHelpers jit(vm, codeBlock);
185
186     // The first thing we need to do is restablish our frame in the case of an exception.
187     if (exit.isGenericUnwindHandler()) {
188         RELEASE_ASSERT(vm->callFrameForCatch); // The first time we hit this exit, like at all other times, this field should be non-null.
189         jit.restoreCalleeSavesFromVMCalleeSavesBuffer();
190         jit.loadPtr(vm->addressOfCallFrameForCatch(), MacroAssembler::framePointerRegister);
191         jit.addPtr(CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
192             MacroAssembler::framePointerRegister, CCallHelpers::stackPointerRegister);
193
194         // Do a pushToSave because that's what the exit compiler below expects the stack
195         // to look like because that's the last thing the ExitThunkGenerator does. The code
196         // below doesn't actually use the value that was pushed, but it does rely on the
197         // general shape of the stack being as it is in the non-exception OSR case.
198         jit.pushToSaveImmediateWithoutTouchingRegisters(CCallHelpers::TrustedImm32(0xbadbeef));
199     }
200
201     // We need scratch space to save all registers, to build up the JS stack, to deal with unwind
202     // fixup, pointers to all of the objects we materialize, and the elements inside those objects
203     // that we materialize.
204     
205     // Figure out how much space we need for those object allocations.
206     unsigned numMaterializations = 0;
207     size_t maxMaterializationNumArguments = 0;
208     for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
209         numMaterializations++;
210         
211         maxMaterializationNumArguments = std::max(
212             maxMaterializationNumArguments,
213             materialization->properties().size());
214     }
215     
216     ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(
217         sizeof(EncodedJSValue) * (
218             exit.m_descriptor->m_values.size() + numMaterializations + maxMaterializationNumArguments) +
219         requiredScratchMemorySizeInBytes() +
220         codeBlock->calleeSaveRegisters()->size() * sizeof(uint64_t));
221     EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
222     EncodedJSValue* materializationPointers = scratch + exit.m_descriptor->m_values.size();
223     EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
224     char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
225     uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
226     
227     HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
228     unsigned materializationCount = 0;
229     for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
230         materializationToPointer.add(
231             materialization, materializationPointers + materializationCount++);
232     }
233
234     auto recoverValue = [&] (const ExitValue& value) {
235         compileRecovery(
236             jit, value,
237             exit.m_valueReps,
238             registerScratch, materializationToPointer);
239     };
240     
241     // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
242     // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
243     // that slot for saveAllRegisters().
244
245     saveAllRegisters(jit, registerScratch);
246     
247     // Bring the stack back into a sane form and assert that it's sane.
248     jit.popToRestore(GPRInfo::regT0);
249     jit.checkStackPointerAlignment();
250     
251     if (vm->m_perBytecodeProfiler && jitCode->dfgCommon()->compilation) {
252         Profiler::Database& database = *vm->m_perBytecodeProfiler;
253         Profiler::Compilation* compilation = jitCode->dfgCommon()->compilation.get();
254         
255         Profiler::OSRExit* profilerExit = compilation->addOSRExit(
256             exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
257             exit.m_kind, exit.m_kind == UncountableInvalidation);
258         jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
259     }
260
261     // The remaining code assumes that SP/FP are in the same state that they were in the FTL's
262     // call frame.
263     
264     // Get the call frame and tag thingies.
265     // Restore the exiting function's callFrame value into a regT4
266     jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
267     jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
268     
269     // Do some value profiling.
270     if (exit.m_descriptor->m_profileDataFormat != DataFormatNone) {
271         Location::forValueRep(exit.m_valueReps[0]).restoreInto(jit, registerScratch, GPRInfo::regT0);
272         reboxAccordingToFormat(
273             exit.m_descriptor->m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
274         
275         if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
276             CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
277             if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
278                 jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
279                 jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
280                 jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
281                 jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
282                 jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
283                 jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
284             }
285         }
286
287         if (!!exit.m_descriptor->m_valueProfile)
288             jit.store64(GPRInfo::regT0, exit.m_descriptor->m_valueProfile.getSpecFailBucket(0));
289     }
290
291     // Materialize all objects. Don't materialize an object until all
292     // of the objects it needs have been materialized. We break cycles
293     // by populating objects late - we only consider an object as
294     // needing another object if the later is needed for the
295     // allocation of the former.
296
297     HashSet<ExitTimeObjectMaterialization*> toMaterialize;
298     for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations)
299         toMaterialize.add(materialization);
300
301     while (!toMaterialize.isEmpty()) {
302         unsigned previousToMaterializeSize = toMaterialize.size();
303
304         Vector<ExitTimeObjectMaterialization*> worklist;
305         worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
306         for (ExitTimeObjectMaterialization* materialization : worklist) {
307             // Check if we can do anything about this right now.
308             bool allGood = true;
309             for (ExitPropertyValue value : materialization->properties()) {
310                 if (!value.value().isObjectMaterialization())
311                     continue;
312                 if (!value.location().neededForMaterialization())
313                     continue;
314                 if (toMaterialize.contains(value.value().objectMaterialization())) {
315                     // Gotta skip this one, since it needs a
316                     // materialization that hasn't been materialized.
317                     allGood = false;
318                     break;
319                 }
320             }
321             if (!allGood)
322                 continue;
323
324             // All systems go for materializing the object. First we
325             // recover the values of all of its fields and then we
326             // call a function to actually allocate the beast.
327             // We only recover the fields that are needed for the allocation.
328             for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
329                 const ExitPropertyValue& property = materialization->properties()[propertyIndex];
330                 if (!property.location().neededForMaterialization())
331                     continue;
332
333                 recoverValue(property.value());
334                 jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
335             }
336             
337             // This call assumes that we don't pass arguments on the stack.
338             jit.setupArgumentsWithExecState(
339                 CCallHelpers::TrustedImmPtr(materialization),
340                 CCallHelpers::TrustedImmPtr(materializationArguments));
341             jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
342             jit.call(GPRInfo::nonArgGPR0);
343             jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
344
345             // Let everyone know that we're done.
346             toMaterialize.remove(materialization);
347         }
348         
349         // We expect progress! This ensures that we crash rather than looping infinitely if there
350         // is something broken about this fixpoint. Or, this could happen if we ever violate the
351         // "materializations form a DAG" rule.
352         RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
353     }
354
355     // Now that all the objects have been allocated, we populate them
356     // with the correct values. This time we can recover all the
357     // fields, including those that are only needed for the allocation.
358     for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
359         for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
360             recoverValue(materialization->properties()[propertyIndex].value());
361             jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
362         }
363
364         // This call assumes that we don't pass arguments on the stack
365         jit.setupArgumentsWithExecState(
366             CCallHelpers::TrustedImmPtr(materialization),
367             CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)),
368             CCallHelpers::TrustedImmPtr(materializationArguments));
369         jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0);
370         jit.call(GPRInfo::nonArgGPR0);
371     }
372
373     // Save all state from wherever the exit data tells us it was, into the appropriate place in
374     // the scratch buffer. This also does the reboxing.
375     
376     for (unsigned index = exit.m_descriptor->m_values.size(); index--;) {
377         recoverValue(exit.m_descriptor->m_values[index]);
378         jit.store64(GPRInfo::regT0, scratch + index);
379     }
380     
381     // Henceforth we make it look like the exiting function was called through a register
382     // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
383     // we restore the various things according to either exit.m_descriptor->m_values or by copying from the
384     // old frame, and finally we save the various callee-save registers into where the
385     // restoration thunk would restore them from.
386     
387     // Before we start messing with the frame, we need to set aside any registers that the
388     // FTL code was preserving.
389     for (unsigned i = codeBlock->calleeSaveRegisters()->size(); i--;) {
390         RegisterAtOffset entry = codeBlock->calleeSaveRegisters()->at(i);
391         jit.load64(
392             MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
393             GPRInfo::regT0);
394         jit.store64(GPRInfo::regT0, unwindScratch + i);
395     }
396     
397     jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
398     
399     // Let's say that the FTL function had failed its arity check. In that case, the stack will
400     // contain some extra stuff.
401     //
402     // We compute the padded stack space:
403     //
404     //     paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
405     //
406     // The stack will have regT2 + CallFrameHeaderSize stuff.
407     // We want to make the stack look like this, from higher addresses down:
408     //
409     //     - argument padding
410     //     - actual arguments
411     //     - call frame header
412
413     // This code assumes that we're dealing with FunctionCode.
414     RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
415     
416     jit.add32(
417         MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
418         GPRInfo::regT3);
419     MacroAssembler::Jump arityIntact = jit.branch32(
420         MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
421     jit.neg32(GPRInfo::regT3);
422     jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
423     jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
424     jit.add32(GPRInfo::regT3, GPRInfo::regT2);
425     arityIntact.link(&jit);
426
427     CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin);
428
429     // First set up SP so that our data doesn't get clobbered by signals.
430     unsigned conservativeStackDelta =
431         (exit.m_descriptor->m_values.numberOfLocals() + baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()) * sizeof(Register) +
432         maxFrameExtentForSlowPathCall;
433     conservativeStackDelta = WTF::roundUpToMultipleOf(
434         stackAlignmentBytes(), conservativeStackDelta);
435     jit.addPtr(
436         MacroAssembler::TrustedImm32(-conservativeStackDelta),
437         MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
438     jit.checkStackPointerAlignment();
439
440     RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters();
441     RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->calleeSaveRegisters();
442     RegisterAtOffsetList* vmCalleeSaves = vm->getAllCalleeSaveRegisterOffsets();
443     RegisterSet vmCalleeSavesToSkip = RegisterSet::stackRegisters();
444     if (exit.isExceptionHandler())
445         jit.move(CCallHelpers::TrustedImmPtr(vm->calleeSaveRegistersBuffer), GPRInfo::regT1);
446
447     for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
448         if (!allFTLCalleeSaves.get(reg)) {
449             if (exit.isExceptionHandler())
450                 RELEASE_ASSERT(!vmCalleeSaves->find(reg));
451             continue;
452         }
453         unsigned unwindIndex = codeBlock->calleeSaveRegisters()->indexOf(reg);
454         RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg);
455         RegisterAtOffset* vmCalleeSave = nullptr; 
456         if (exit.isExceptionHandler())
457             vmCalleeSave = vmCalleeSaves->find(reg);
458
459         if (reg.isGPR()) {
460             GPRReg regToLoad = baselineRegisterOffset ? GPRInfo::regT0 : reg.gpr();
461             RELEASE_ASSERT(regToLoad != GPRInfo::regT1);
462
463             if (unwindIndex == UINT_MAX) {
464                 // The FTL compilation didn't preserve this register. This means that it also
465                 // didn't use the register. So its value at the beginning of OSR exit should be
466                 // preserved by the thunk. Luckily, we saved all registers into the register
467                 // scratch buffer, so we can restore them from there.
468                 jit.load64(registerScratch + offsetOfReg(reg), regToLoad);
469             } else {
470                 // The FTL compilation preserved the register. Its new value is therefore
471                 // irrelevant, but we can get the value that was preserved by using the unwind
472                 // data. We've already copied all unwind-able preserved registers into the unwind
473                 // scratch buffer, so we can get it from there.
474                 jit.load64(unwindScratch + unwindIndex, regToLoad);
475             }
476
477             if (baselineRegisterOffset)
478                 jit.store64(regToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
479             if (vmCalleeSave && !vmCalleeSavesToSkip.get(vmCalleeSave->reg()))
480                 jit.store64(regToLoad, MacroAssembler::Address(GPRInfo::regT1, vmCalleeSave->offset()));
481         } else {
482             FPRReg fpRegToLoad = baselineRegisterOffset ? FPRInfo::fpRegT0 : reg.fpr();
483
484             if (unwindIndex == UINT_MAX)
485                 jit.loadDouble(MacroAssembler::TrustedImmPtr(registerScratch + offsetOfReg(reg)), fpRegToLoad);
486             else
487                 jit.loadDouble(MacroAssembler::TrustedImmPtr(unwindScratch + unwindIndex), fpRegToLoad);
488
489             if (baselineRegisterOffset)
490                 jit.storeDouble(fpRegToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
491             if (vmCalleeSave && !vmCalleeSavesToSkip.get(vmCalleeSave->reg()))
492                 jit.storeDouble(fpRegToLoad, MacroAssembler::Address(GPRInfo::regT1, vmCalleeSave->offset()));
493         }
494     }
495
496     if (exit.isExceptionHandler()) {
497         RegisterAtOffset* vmCalleeSave = vmCalleeSaves->find(GPRInfo::tagTypeNumberRegister);
498         jit.store64(GPRInfo::tagTypeNumberRegister, MacroAssembler::Address(GPRInfo::regT1, vmCalleeSave->offset()));
499
500         vmCalleeSave = vmCalleeSaves->find(GPRInfo::tagMaskRegister);
501         jit.store64(GPRInfo::tagMaskRegister, MacroAssembler::Address(GPRInfo::regT1, vmCalleeSave->offset()));
502     }
503
504     size_t baselineVirtualRegistersForCalleeSaves = baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters();
505
506     // Now get state out of the scratch buffer and place it back into the stack. The values are
507     // already reboxed so we just move them.
508     for (unsigned index = exit.m_descriptor->m_values.size(); index--;) {
509         VirtualRegister reg = exit.m_descriptor->m_values.virtualRegisterForIndex(index);
510
511         if (reg.isLocal() && reg.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves))
512             continue;
513
514         jit.load64(scratch + index, GPRInfo::regT0);
515         jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(reg));
516     }
517     
518     handleExitCounts(jit, exit);
519     reifyInlinedCallFrames(jit, exit);
520     adjustAndJumpToTarget(jit, exit);
521     
522     LinkBuffer patchBuffer(*vm, jit, codeBlock);
523     exit.m_code = FINALIZE_CODE_IF(
524         shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
525         patchBuffer,
526         ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s",
527             exitID, toCString(exit.m_codeOrigin).data(),
528             exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
529             toCString(ignoringContext<DumpContext>(exit.m_descriptor->m_values)).data())
530         );
531 }
532
533 extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
534 {
535     SamplingRegion samplingRegion("FTL OSR Exit Compilation");
536
537     if (shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
538         dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
539
540     if (exec->vm().callFrameForCatch)
541         RELEASE_ASSERT(exec->vm().callFrameForCatch == exec);
542     
543     CodeBlock* codeBlock = exec->codeBlock();
544     
545     ASSERT(codeBlock);
546     ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
547     
548     VM* vm = &exec->vm();
549     
550     // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
551     // really be profitable.
552     DeferGCForAWhile deferGC(vm->heap);
553
554     JITCode* jitCode = codeBlock->jitCode()->ftl();
555     OSRExit& exit = jitCode->osrExit[exitID];
556     
557     if (shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit()) {
558         dataLog("    Owning block: ", pointerDump(codeBlock), "\n");
559         dataLog("    Origin: ", exit.m_codeOrigin, "\n");
560         if (exit.m_codeOriginForExitProfile != exit.m_codeOrigin)
561             dataLog("    Origin for exit profile: ", exit.m_codeOriginForExitProfile, "\n");
562         dataLog("    Current call site index: ", exec->callSiteIndex().bits(), "\n");
563         dataLog("    Exit is exception handler: ", exit.isExceptionHandler(), "\n");
564         dataLog("    Is unwind handler: ", exit.isGenericUnwindHandler(), "\n");
565         dataLog("    Exit values: ", exit.m_descriptor->m_values, "\n");
566         dataLog("    Value reps: ", listDump(exit.m_valueReps), "\n");
567         if (!exit.m_descriptor->m_materializations.isEmpty()) {
568             dataLog("    Materializations:\n");
569             for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations)
570                 dataLog("        ", pointerDump(materialization), "\n");
571         }
572     }
573
574     prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
575     
576     compileStub(exitID, jitCode, exit, vm, codeBlock);
577
578     MacroAssembler::repatchJump(
579         exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
580     
581     return exit.m_code.code().executableAddress();
582 }
583
584 } } // namespace JSC::FTL
585
586 #endif // ENABLE(FTL_JIT)
587