Move DFG::OSRExitCompiler methods into DFG::OSRExit [step 3].
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGOSRExit.cpp
1 /*
2  * Copyright (C) 2011, 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGOSRExit.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "AssemblyHelpers.h"
32 #include "DFGGraph.h"
33 #include "DFGMayExit.h"
34 #include "DFGOSRExitCompilerCommon.h"
35 #include "DFGOSRExitPreparation.h"
36 #include "DFGOperations.h"
37 #include "DFGSpeculativeJIT.h"
38 #include "FrameTracers.h"
39 #include "JSCInlines.h"
40 #include "OperandsInlines.h"
41
42 namespace JSC { namespace DFG {
43
44 OSRExit::OSRExit(ExitKind kind, JSValueSource jsValueSource, MethodOfGettingAValueProfile valueProfile, SpeculativeJIT* jit, unsigned streamIndex, unsigned recoveryIndex)
45     : OSRExitBase(kind, jit->m_origin.forExit, jit->m_origin.semantic, jit->m_origin.wasHoisted)
46     , m_jsValueSource(jsValueSource)
47     , m_valueProfile(valueProfile)
48     , m_recoveryIndex(recoveryIndex)
49     , m_streamIndex(streamIndex)
50 {
51     bool canExit = jit->m_origin.exitOK;
52     if (!canExit && jit->m_currentNode) {
53         ExitMode exitMode = mayExit(jit->m_jit.graph(), jit->m_currentNode);
54         canExit = exitMode == ExitMode::Exits || exitMode == ExitMode::ExitsForExceptions;
55     }
56     DFG_ASSERT(jit->m_jit.graph(), jit->m_currentNode, canExit);
57 }
58
59 void OSRExit::setPatchableCodeOffset(MacroAssembler::PatchableJump check)
60 {
61     m_patchableCodeOffset = check.m_jump.m_label.m_offset;
62 }
63
64 MacroAssembler::Jump OSRExit::getPatchableCodeOffsetAsJump() const
65 {
66     return MacroAssembler::Jump(AssemblerLabel(m_patchableCodeOffset));
67 }
68
69 CodeLocationJump OSRExit::codeLocationForRepatch(CodeBlock* dfgCodeBlock) const
70 {
71     return CodeLocationJump(dfgCodeBlock->jitCode()->dataAddressAtOffset(m_patchableCodeOffset));
72 }
73
74 void OSRExit::correctJump(LinkBuffer& linkBuffer)
75 {
76     MacroAssembler::Label label;
77     label.m_label.m_offset = m_patchableCodeOffset;
78     m_patchableCodeOffset = linkBuffer.offsetOf(label);
79 }
80
81 void OSRExit::emitRestoreArguments(CCallHelpers& jit, const Operands<ValueRecovery>& operands)
82 {
83     HashMap<MinifiedID, int> alreadyAllocatedArguments; // Maps phantom arguments node ID to operand.
84     for (size_t index = 0; index < operands.size(); ++index) {
85         const ValueRecovery& recovery = operands[index];
86         int operand = operands.operandForIndex(index);
87
88         if (recovery.technique() != DirectArgumentsThatWereNotCreated
89             && recovery.technique() != ClonedArgumentsThatWereNotCreated)
90             continue;
91
92         MinifiedID id = recovery.nodeID();
93         auto iter = alreadyAllocatedArguments.find(id);
94         if (iter != alreadyAllocatedArguments.end()) {
95             JSValueRegs regs = JSValueRegs::withTwoAvailableRegs(GPRInfo::regT0, GPRInfo::regT1);
96             jit.loadValue(CCallHelpers::addressFor(iter->value), regs);
97             jit.storeValue(regs, CCallHelpers::addressFor(operand));
98             continue;
99         }
100
101         InlineCallFrame* inlineCallFrame =
102             jit.codeBlock()->jitCode()->dfg()->minifiedDFG.at(id)->inlineCallFrame();
103
104         int stackOffset;
105         if (inlineCallFrame)
106             stackOffset = inlineCallFrame->stackOffset;
107         else
108             stackOffset = 0;
109
110         if (!inlineCallFrame || inlineCallFrame->isClosureCall) {
111             jit.loadPtr(
112                 AssemblyHelpers::addressFor(stackOffset + CallFrameSlot::callee),
113                 GPRInfo::regT0);
114         } else {
115             jit.move(
116                 AssemblyHelpers::TrustedImmPtr(inlineCallFrame->calleeRecovery.constant().asCell()),
117                 GPRInfo::regT0);
118         }
119
120         if (!inlineCallFrame || inlineCallFrame->isVarargs()) {
121             jit.load32(
122                 AssemblyHelpers::payloadFor(stackOffset + CallFrameSlot::argumentCount),
123                 GPRInfo::regT1);
124         } else {
125             jit.move(
126                 AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()),
127                 GPRInfo::regT1);
128         }
129
130         jit.setupArgumentsWithExecState(
131             AssemblyHelpers::TrustedImmPtr(inlineCallFrame), GPRInfo::regT0, GPRInfo::regT1);
132         switch (recovery.technique()) {
133         case DirectArgumentsThatWereNotCreated:
134             jit.move(AssemblyHelpers::TrustedImmPtr(bitwise_cast<void*>(operationCreateDirectArgumentsDuringExit)), GPRInfo::nonArgGPR0);
135             break;
136         case ClonedArgumentsThatWereNotCreated:
137             jit.move(AssemblyHelpers::TrustedImmPtr(bitwise_cast<void*>(operationCreateClonedArgumentsDuringExit)), GPRInfo::nonArgGPR0);
138             break;
139         default:
140             RELEASE_ASSERT_NOT_REACHED();
141             break;
142         }
143         jit.call(GPRInfo::nonArgGPR0);
144         jit.storeCell(GPRInfo::returnValueGPR, AssemblyHelpers::addressFor(operand));
145
146         alreadyAllocatedArguments.add(id, operand);
147     }
148 }
149
150 void JIT_OPERATION OSRExit::compileOSRExit(ExecState* exec)
151 {
152     VM* vm = &exec->vm();
153     auto scope = DECLARE_THROW_SCOPE(*vm);
154
155     if (vm->callFrameForCatch)
156         RELEASE_ASSERT(vm->callFrameForCatch == exec);
157
158     CodeBlock* codeBlock = exec->codeBlock();
159     ASSERT(codeBlock);
160     ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
161
162     // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
163     // really be profitable.
164     DeferGCForAWhile deferGC(vm->heap);
165
166     uint32_t exitIndex = vm->osrExitIndex;
167     OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
168
169     if (vm->callFrameForCatch)
170         ASSERT(exit.m_kind == GenericUnwind);
171     if (exit.isExceptionHandler())
172         ASSERT_UNUSED(scope, !!scope.exception());
173     
174     prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
175
176     // Compute the value recoveries.
177     Operands<ValueRecovery> operands;
178     codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
179
180     SpeculationRecovery* recovery = 0;
181     if (exit.m_recoveryIndex != UINT_MAX)
182         recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
183
184     {
185         CCallHelpers jit(codeBlock);
186
187         if (exit.m_kind == GenericUnwind) {
188             // We are acting as a defacto op_catch because we arrive here from genericUnwind().
189             // So, we must restore our call frame and stack pointer.
190             jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm);
191             jit.loadPtr(vm->addressOfCallFrameForCatch(), GPRInfo::callFrameRegister);
192         }
193         jit.addPtr(
194             CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
195             GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
196
197         jit.jitAssertHasValidCallFrame();
198
199         if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
200             Profiler::Database& database = *vm->m_perBytecodeProfiler;
201             Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
202
203             Profiler::OSRExit* profilerExit = compilation->addOSRExit(
204                 exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
205                 exit.m_kind, exit.m_kind == UncountableInvalidation);
206             jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
207         }
208
209         compileExit(jit, *vm, exit, operands, recovery);
210
211         LinkBuffer patchBuffer(jit, codeBlock);
212         exit.m_code = FINALIZE_CODE_IF(
213             shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseDFGOSRExit(),
214             patchBuffer,
215             ("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
216                 exitIndex, toCString(exit.m_codeOrigin).data(),
217                 exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
218                 toCString(ignoringContext<DumpContext>(operands)).data()));
219     }
220
221     MacroAssembler::repatchJump(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
222
223     vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
224 }
225
226 void OSRExit::compileExit(CCallHelpers& jit, VM& vm, const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery)
227 {
228     jit.jitAssertTagsInPlace();
229
230     // Pro-forma stuff.
231     if (Options::printEachOSRExit()) {
232         SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
233         debugInfo->codeBlock = jit.codeBlock();
234         debugInfo->kind = exit.m_kind;
235         debugInfo->bytecodeOffset = exit.m_codeOrigin.bytecodeIndex;
236
237         jit.debugCall(vm, debugOperationPrintSpeculationFailure, debugInfo);
238     }
239
240     // Perform speculation recovery. This only comes into play when an operation
241     // starts mutating state before verifying the speculation it has already made.
242
243     if (recovery) {
244         switch (recovery->type()) {
245         case SpeculativeAdd:
246             jit.sub32(recovery->src(), recovery->dest());
247 #if USE(JSVALUE64)
248             jit.or64(GPRInfo::tagTypeNumberRegister, recovery->dest());
249 #endif
250             break;
251
252         case SpeculativeAddImmediate:
253             jit.sub32(AssemblyHelpers::Imm32(recovery->immediate()), recovery->dest());
254 #if USE(JSVALUE64)
255             jit.or64(GPRInfo::tagTypeNumberRegister, recovery->dest());
256 #endif
257             break;
258
259         case BooleanSpeculationCheck:
260 #if USE(JSVALUE64)
261             jit.xor64(AssemblyHelpers::TrustedImm32(static_cast<int32_t>(ValueFalse)), recovery->dest());
262 #endif
263             break;
264
265         default:
266             break;
267         }
268     }
269
270     // Refine some array and/or value profile, if appropriate.
271
272     if (!!exit.m_jsValueSource) {
273         if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
274             // If the instruction that this originated from has an array profile, then
275             // refine it. If it doesn't, then do nothing. The latter could happen for
276             // hoisted checks, or checks emitted for operations that didn't have array
277             // profiling - either ops that aren't array accesses at all, or weren't
278             // known to be array acceses in the bytecode. The latter case is a FIXME
279             // while the former case is an outcome of a CheckStructure not knowing why
280             // it was emitted (could be either due to an inline cache of a property
281             // property access, or due to an array profile).
282
283             CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
284             if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
285 #if USE(JSVALUE64)
286                 GPRReg usedRegister;
287                 if (exit.m_jsValueSource.isAddress())
288                     usedRegister = exit.m_jsValueSource.base();
289                 else
290                     usedRegister = exit.m_jsValueSource.gpr();
291 #else
292                 GPRReg usedRegister1;
293                 GPRReg usedRegister2;
294                 if (exit.m_jsValueSource.isAddress()) {
295                     usedRegister1 = exit.m_jsValueSource.base();
296                     usedRegister2 = InvalidGPRReg;
297                 } else {
298                     usedRegister1 = exit.m_jsValueSource.payloadGPR();
299                     if (exit.m_jsValueSource.hasKnownTag())
300                         usedRegister2 = InvalidGPRReg;
301                     else
302                         usedRegister2 = exit.m_jsValueSource.tagGPR();
303                 }
304 #endif
305
306                 GPRReg scratch1;
307                 GPRReg scratch2;
308 #if USE(JSVALUE64)
309                 scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister);
310                 scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister, scratch1);
311 #else
312                 scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2);
313                 scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2, scratch1);
314 #endif
315
316                 if (isARM64()) {
317                     jit.pushToSave(scratch1);
318                     jit.pushToSave(scratch2);
319                 } else {
320                     jit.push(scratch1);
321                     jit.push(scratch2);
322                 }
323
324                 GPRReg value;
325                 if (exit.m_jsValueSource.isAddress()) {
326                     value = scratch1;
327                     jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value);
328                 } else
329                     value = exit.m_jsValueSource.payloadGPR();
330
331                 jit.load32(AssemblyHelpers::Address(value, JSCell::structureIDOffset()), scratch1);
332                 jit.store32(scratch1, arrayProfile->addressOfLastSeenStructureID());
333 #if USE(JSVALUE64)
334                 jit.load8(AssemblyHelpers::Address(value, JSCell::indexingTypeAndMiscOffset()), scratch1);
335 #else
336                 jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingTypeIncludingHistoryOffset()), scratch1);
337 #endif
338                 jit.move(AssemblyHelpers::TrustedImm32(1), scratch2);
339                 jit.lshift32(scratch1, scratch2);
340                 jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
341
342                 if (isARM64()) {
343                     jit.popToRestore(scratch2);
344                     jit.popToRestore(scratch1);
345                 } else {
346                     jit.pop(scratch2);
347                     jit.pop(scratch1);
348                 }
349             }
350         }
351
352         if (MethodOfGettingAValueProfile profile = exit.m_valueProfile) {
353 #if USE(JSVALUE64)
354             if (exit.m_jsValueSource.isAddress()) {
355                 // We can't be sure that we have a spare register. So use the tagTypeNumberRegister,
356                 // since we know how to restore it.
357                 jit.load64(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::tagTypeNumberRegister);
358                 profile.emitReportValue(jit, JSValueRegs(GPRInfo::tagTypeNumberRegister));
359                 jit.move(AssemblyHelpers::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
360             } else
361                 profile.emitReportValue(jit, JSValueRegs(exit.m_jsValueSource.gpr()));
362 #else // not USE(JSVALUE64)
363             if (exit.m_jsValueSource.isAddress()) {
364                 // Save a register so we can use it.
365                 GPRReg scratchPayload = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base());
366                 GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base(), scratchPayload);
367                 jit.pushToSave(scratchPayload);
368                 jit.pushToSave(scratchTag);
369
370                 JSValueRegs scratch(scratchTag, scratchPayload);
371                 
372                 jit.loadValue(exit.m_jsValueSource.asAddress(), scratch);
373                 profile.emitReportValue(jit, scratch);
374                 
375                 jit.popToRestore(scratchTag);
376                 jit.popToRestore(scratchPayload);
377             } else if (exit.m_jsValueSource.hasKnownTag()) {
378                 GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.payloadGPR());
379                 jit.pushToSave(scratchTag);
380                 jit.move(AssemblyHelpers::TrustedImm32(exit.m_jsValueSource.tag()), scratchTag);
381                 JSValueRegs value(scratchTag, exit.m_jsValueSource.payloadGPR());
382                 profile.emitReportValue(jit, value);
383                 jit.popToRestore(scratchTag);
384             } else
385                 profile.emitReportValue(jit, exit.m_jsValueSource.regs());
386 #endif // USE(JSVALUE64)
387         }
388     }
389
390     // What follows is an intentionally simple OSR exit implementation that generates
391     // fairly poor code but is very easy to hack. In particular, it dumps all state that
392     // needs conversion into a scratch buffer so that in step 6, where we actually do the
393     // conversions, we know that all temp registers are free to use and the variable is
394     // definitely in a well-known spot in the scratch buffer regardless of whether it had
395     // originally been in a register or spilled. This allows us to decouple "where was
396     // the variable" from "how was it represented". Consider that the
397     // Int32DisplacedInJSStack recovery: it tells us that the value is in a
398     // particular place and that that place holds an unboxed int32. We have two different
399     // places that a value could be (displaced, register) and a bunch of different
400     // ways of representing a value. The number of recoveries is two * a bunch. The code
401     // below means that we have to have two + a bunch cases rather than two * a bunch.
402     // Once we have loaded the value from wherever it was, the reboxing is the same
403     // regardless of its location. Likewise, before we do the reboxing, the way we get to
404     // the value (i.e. where we load it from) is the same regardless of its type. Because
405     // the code below always dumps everything into a scratch buffer first, the two
406     // questions become orthogonal, which simplifies adding new types and adding new
407     // locations.
408     //
409     // This raises the question: does using such a suboptimal implementation of OSR exit,
410     // where we always emit code to dump all state into a scratch buffer only to then
411     // dump it right back into the stack, hurt us in any way? The asnwer is that OSR exits
412     // are rare. Our tiering strategy ensures this. This is because if an OSR exit is
413     // taken more than ~100 times, we jettison the DFG code block along with all of its
414     // exits. It is impossible for an OSR exit - i.e. the code we compile below - to
415     // execute frequently enough for the codegen to matter that much. It probably matters
416     // enough that we don't want to turn this into some super-slow function call, but so
417     // long as we're generating straight-line code, that code can be pretty bad. Also
418     // because we tend to exit only along one OSR exit from any DFG code block - that's an
419     // empirical result that we're extremely confident about - the code size of this
420     // doesn't matter much. Hence any attempt to optimize the codegen here is just purely
421     // harmful to the system: it probably won't reduce either net memory usage or net
422     // execution time. It will only prevent us from cleanly decoupling "where was the
423     // variable" from "how was it represented", which will make it more difficult to add
424     // features in the future and it will make it harder to reason about bugs.
425
426     // Save all state from GPRs into the scratch buffer.
427
428     ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(sizeof(EncodedJSValue) * operands.size());
429     EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
430
431     for (size_t index = 0; index < operands.size(); ++index) {
432         const ValueRecovery& recovery = operands[index];
433
434         switch (recovery.technique()) {
435         case UnboxedInt32InGPR:
436         case UnboxedCellInGPR:
437 #if USE(JSVALUE64)
438         case InGPR:
439         case UnboxedInt52InGPR:
440         case UnboxedStrictInt52InGPR:
441             jit.store64(recovery.gpr(), scratch + index);
442             break;
443 #else
444         case UnboxedBooleanInGPR:
445             jit.store32(
446                 recovery.gpr(),
447                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
448             break;
449             
450         case InPair:
451             jit.store32(
452                 recovery.tagGPR(),
453                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
454             jit.store32(
455                 recovery.payloadGPR(),
456                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
457             break;
458 #endif
459
460         default:
461             break;
462         }
463     }
464
465     // And voila, all GPRs are free to reuse.
466
467     // Save all state from FPRs into the scratch buffer.
468
469     for (size_t index = 0; index < operands.size(); ++index) {
470         const ValueRecovery& recovery = operands[index];
471
472         switch (recovery.technique()) {
473         case UnboxedDoubleInFPR:
474         case InFPR:
475             jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
476             jit.storeDouble(recovery.fpr(), MacroAssembler::Address(GPRInfo::regT0));
477             break;
478
479         default:
480             break;
481         }
482     }
483
484     // Now, all FPRs are also free.
485
486     // Save all state from the stack into the scratch buffer. For simplicity we
487     // do this even for state that's already in the right place on the stack.
488     // It makes things simpler later.
489
490     for (size_t index = 0; index < operands.size(); ++index) {
491         const ValueRecovery& recovery = operands[index];
492
493         switch (recovery.technique()) {
494         case DisplacedInJSStack:
495         case CellDisplacedInJSStack:
496         case BooleanDisplacedInJSStack:
497         case Int32DisplacedInJSStack:
498         case DoubleDisplacedInJSStack:
499 #if USE(JSVALUE64)
500         case Int52DisplacedInJSStack:
501         case StrictInt52DisplacedInJSStack:
502             jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
503             jit.store64(GPRInfo::regT0, scratch + index);
504             break;
505 #else
506             jit.load32(
507                 AssemblyHelpers::tagFor(recovery.virtualRegister()),
508                 GPRInfo::regT0);
509             jit.load32(
510                 AssemblyHelpers::payloadFor(recovery.virtualRegister()),
511                 GPRInfo::regT1);
512             jit.store32(
513                 GPRInfo::regT0,
514                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
515             jit.store32(
516                 GPRInfo::regT1,
517                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
518             break;
519 #endif
520
521         default:
522             break;
523         }
524     }
525
526     // Need to ensure that the stack pointer accounts for the worst-case stack usage at exit. This
527     // could toast some stack that the DFG used. We need to do it before storing to stack offsets
528     // used by baseline.
529     jit.addPtr(
530         CCallHelpers::TrustedImm32(
531             -jit.codeBlock()->jitCode()->dfgCommon()->requiredRegisterCountForExit * sizeof(Register)),
532         CCallHelpers::framePointerRegister, CCallHelpers::stackPointerRegister);
533
534     // Restore the DFG callee saves and then save the ones the baseline JIT uses.
535     jit.emitRestoreCalleeSaves();
536     jit.emitSaveCalleeSavesFor(jit.baselineCodeBlock());
537
538     // The tag registers are needed to materialize recoveries below.
539     jit.emitMaterializeTagCheckRegisters();
540
541     if (exit.isExceptionHandler())
542         jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(vm);
543
544     // Do all data format conversions and store the results into the stack.
545
546     for (size_t index = 0; index < operands.size(); ++index) {
547         const ValueRecovery& recovery = operands[index];
548         VirtualRegister reg = operands.virtualRegisterForIndex(index);
549
550         if (reg.isLocal() && reg.toLocal() < static_cast<int>(jit.baselineCodeBlock()->calleeSaveSpaceAsVirtualRegisters()))
551             continue;
552
553         int operand = reg.offset();
554
555         switch (recovery.technique()) {
556         case DisplacedInJSStack:
557         case InFPR:
558 #if USE(JSVALUE64)
559         case InGPR:
560         case UnboxedCellInGPR:
561         case CellDisplacedInJSStack:
562         case BooleanDisplacedInJSStack:
563             jit.load64(scratch + index, GPRInfo::regT0);
564             jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
565             break;
566 #else // not USE(JSVALUE64)
567         case InPair:
568             jit.load32(
569                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag,
570                 GPRInfo::regT0);
571             jit.load32(
572                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
573                 GPRInfo::regT1);
574             jit.store32(
575                 GPRInfo::regT0,
576                 AssemblyHelpers::tagFor(operand));
577             jit.store32(
578                 GPRInfo::regT1,
579                 AssemblyHelpers::payloadFor(operand));
580             break;
581
582         case UnboxedCellInGPR:
583         case CellDisplacedInJSStack:
584             jit.load32(
585                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
586                 GPRInfo::regT0);
587             jit.store32(
588                 AssemblyHelpers::TrustedImm32(JSValue::CellTag),
589                 AssemblyHelpers::tagFor(operand));
590             jit.store32(
591                 GPRInfo::regT0,
592                 AssemblyHelpers::payloadFor(operand));
593             break;
594
595         case UnboxedBooleanInGPR:
596         case BooleanDisplacedInJSStack:
597             jit.load32(
598                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
599                 GPRInfo::regT0);
600             jit.store32(
601                 AssemblyHelpers::TrustedImm32(JSValue::BooleanTag),
602                 AssemblyHelpers::tagFor(operand));
603             jit.store32(
604                 GPRInfo::regT0,
605                 AssemblyHelpers::payloadFor(operand));
606             break;
607 #endif // USE(JSVALUE64)
608
609         case UnboxedInt32InGPR:
610         case Int32DisplacedInJSStack:
611 #if USE(JSVALUE64)
612             jit.load64(scratch + index, GPRInfo::regT0);
613             jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0);
614             jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
615             jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
616 #else
617             jit.load32(
618                 &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
619                 GPRInfo::regT0);
620             jit.store32(
621                 AssemblyHelpers::TrustedImm32(JSValue::Int32Tag),
622                 AssemblyHelpers::tagFor(operand));
623             jit.store32(
624                 GPRInfo::regT0,
625                 AssemblyHelpers::payloadFor(operand));
626 #endif
627             break;
628
629 #if USE(JSVALUE64)
630         case UnboxedInt52InGPR:
631         case Int52DisplacedInJSStack:
632             jit.load64(scratch + index, GPRInfo::regT0);
633             jit.rshift64(
634                 AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0);
635             jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
636             jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
637             break;
638
639         case UnboxedStrictInt52InGPR:
640         case StrictInt52DisplacedInJSStack:
641             jit.load64(scratch + index, GPRInfo::regT0);
642             jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
643             jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
644             break;
645 #endif
646
647         case UnboxedDoubleInFPR:
648         case DoubleDisplacedInJSStack:
649             jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
650             jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::fpRegT0);
651             jit.purifyNaN(FPRInfo::fpRegT0);
652 #if USE(JSVALUE64)
653             jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
654             jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
655 #else
656             jit.storeDouble(FPRInfo::fpRegT0, AssemblyHelpers::addressFor(operand));
657 #endif
658             break;
659
660         case Constant:
661 #if USE(JSVALUE64)
662             jit.store64(
663                 AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())),
664                 AssemblyHelpers::addressFor(operand));
665 #else
666             jit.store32(
667                 AssemblyHelpers::TrustedImm32(recovery.constant().tag()),
668                 AssemblyHelpers::tagFor(operand));
669             jit.store32(
670                 AssemblyHelpers::TrustedImm32(recovery.constant().payload()),
671                 AssemblyHelpers::payloadFor(operand));
672 #endif
673             break;
674
675         case DirectArgumentsThatWereNotCreated:
676         case ClonedArgumentsThatWereNotCreated:
677             // Don't do this, yet.
678             break;
679
680         default:
681             RELEASE_ASSERT_NOT_REACHED();
682             break;
683         }
684     }
685
686     // Now that things on the stack are recovered, do the arguments recovery. We assume that arguments
687     // recoveries don't recursively refer to each other. But, we don't try to assume that they only
688     // refer to certain ranges of locals. Hence why we need to do this here, once the stack is sensible.
689     // Note that we also roughly assume that the arguments might still be materialized outside of its
690     // inline call frame scope - but for now the DFG wouldn't do that.
691
692     emitRestoreArguments(jit, operands);
693
694     // Adjust the old JIT's execute counter. Since we are exiting OSR, we know
695     // that all new calls into this code will go to the new JIT, so the execute
696     // counter only affects call frames that performed OSR exit and call frames
697     // that were still executing the old JIT at the time of another call frame's
698     // OSR exit. We want to ensure that the following is true:
699     //
700     // (a) Code the performs an OSR exit gets a chance to reenter optimized
701     //     code eventually, since optimized code is faster. But we don't
702     //     want to do such reentery too aggressively (see (c) below).
703     //
704     // (b) If there is code on the call stack that is still running the old
705     //     JIT's code and has never OSR'd, then it should get a chance to
706     //     perform OSR entry despite the fact that we've exited.
707     //
708     // (c) Code the performs an OSR exit should not immediately retry OSR
709     //     entry, since both forms of OSR are expensive. OSR entry is
710     //     particularly expensive.
711     //
712     // (d) Frequent OSR failures, even those that do not result in the code
713     //     running in a hot loop, result in recompilation getting triggered.
714     //
715     // To ensure (c), we'd like to set the execute counter to
716     // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
717     // (a) and (b), since then every OSR exit would delay the opportunity for
718     // every call frame to perform OSR entry. Essentially, if OSR exit happens
719     // frequently and the function has few loops, then the counter will never
720     // become non-negative and OSR entry will never be triggered. OSR entry
721     // will only happen if a loop gets hot in the old JIT, which does a pretty
722     // good job of ensuring (a) and (b). But that doesn't take care of (d),
723     // since each speculation failure would reset the execute counter.
724     // So we check here if the number of speculation failures is significantly
725     // larger than the number of successes (we want 90% success rate), and if
726     // there have been a large enough number of failures. If so, we set the
727     // counter to 0; otherwise we set the counter to
728     // counterValueForOptimizeAfterWarmUp().
729
730     handleExitCounts(jit, exit);
731
732     // Reify inlined call frames.
733
734     reifyInlinedCallFrames(jit, exit);
735
736     // And finish.
737     adjustAndJumpToTarget(vm, jit, exit);
738 }
739
740 void JIT_OPERATION OSRExit::debugOperationPrintSpeculationFailure(ExecState* exec, void* debugInfoRaw, void* scratch)
741 {
742     VM* vm = &exec->vm();
743     NativeCallFrameTracer tracer(vm, exec);
744
745     SpeculationFailureDebugInfo* debugInfo = static_cast<SpeculationFailureDebugInfo*>(debugInfoRaw);
746     CodeBlock* codeBlock = debugInfo->codeBlock;
747     CodeBlock* alternative = codeBlock->alternative();
748     dataLog("Speculation failure in ", *codeBlock);
749     dataLog(" @ exit #", vm->osrExitIndex, " (bc#", debugInfo->bytecodeOffset, ", ", exitKindToString(debugInfo->kind), ") with ");
750     if (alternative) {
751         dataLog(
752             "executeCounter = ", alternative->jitExecuteCounter(),
753             ", reoptimizationRetryCounter = ", alternative->reoptimizationRetryCounter(),
754             ", optimizationDelayCounter = ", alternative->optimizationDelayCounter());
755     } else
756         dataLog("no alternative code block (i.e. we've been jettisoned)");
757     dataLog(", osrExitCounter = ", codeBlock->osrExitCounter(), "\n");
758     dataLog("    GPRs at time of exit:");
759     char* scratchPointer = static_cast<char*>(scratch);
760     for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
761         GPRReg gpr = GPRInfo::toRegister(i);
762         dataLog(" ", GPRInfo::debugName(gpr), ":", RawPointer(*reinterpret_cast_ptr<void**>(scratchPointer)));
763         scratchPointer += sizeof(EncodedJSValue);
764     }
765     dataLog("\n");
766     dataLog("    FPRs at time of exit:");
767     for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
768         FPRReg fpr = FPRInfo::toRegister(i);
769         dataLog(" ", FPRInfo::debugName(fpr), ":");
770         uint64_t bits = *reinterpret_cast_ptr<uint64_t*>(scratchPointer);
771         double value = *reinterpret_cast_ptr<double*>(scratchPointer);
772         dataLogF("%llx:%lf", static_cast<long long>(bits), value);
773         scratchPointer += sizeof(EncodedJSValue);
774     }
775     dataLog("\n");
776 }
777
778 } } // namespace JSC::DFG
779
780 #endif // ENABLE(DFG_JIT)