REGRESSION(r222563): removed DoubleReal type check causes tons of crashes because...
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCapabilities.h"
37 #include "DFGMayExit.h"
38 #include "DFGOSRExitFuzz.h"
39 #include "DFGSaneStringGetByValSlowPathGenerator.h"
40 #include "DFGSlowPathGenerator.h"
41 #include "DFGSnippetParams.h"
42 #include "DirectArguments.h"
43 #include "JITAddGenerator.h"
44 #include "JITBitAndGenerator.h"
45 #include "JITBitOrGenerator.h"
46 #include "JITBitXorGenerator.h"
47 #include "JITDivGenerator.h"
48 #include "JITLeftShiftGenerator.h"
49 #include "JITMulGenerator.h"
50 #include "JITRightShiftGenerator.h"
51 #include "JITSubGenerator.h"
52 #include "JSAsyncFunction.h"
53 #include "JSAsyncGeneratorFunction.h"
54 #include "JSCInlines.h"
55 #include "JSFixedArray.h"
56 #include "JSGeneratorFunction.h"
57 #include "JSLexicalEnvironment.h"
58 #include "JSPropertyNameEnumerator.h"
59 #include "LinkBuffer.h"
60 #include "RegExpConstructor.h"
61 #include "ScopedArguments.h"
62 #include "ScratchRegisterAllocator.h"
63 #include "SuperSampler.h"
64 #include "WeakMapImpl.h"
65 #include <wtf/BitVector.h>
66 #include <wtf/Box.h>
67 #include <wtf/MathExtras.h>
68
69 namespace JSC { namespace DFG {
70
71 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
72     : m_compileOkay(true)
73     , m_jit(jit)
74     , m_graph(m_jit.graph())
75     , m_currentNode(0)
76     , m_lastGeneratedNode(LastNodeType)
77     , m_indexInBlock(0)
78     , m_generationInfo(m_jit.graph().frameRegisterCount())
79     , m_state(m_jit.graph())
80     , m_interpreter(m_jit.graph(), m_state)
81     , m_stream(&jit.jitCode()->variableEventStream)
82     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
83 {
84 }
85
86 SpeculativeJIT::~SpeculativeJIT()
87 {
88 }
89
90 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
91 {
92     IndexingType indexingType = structure->indexingType();
93     bool hasIndexingHeader = hasIndexedProperties(indexingType);
94
95     unsigned inlineCapacity = structure->inlineCapacity();
96     unsigned outOfLineCapacity = structure->outOfLineCapacity();
97     
98     GPRTemporary scratch(this);
99     GPRTemporary scratch2(this);
100     GPRReg scratchGPR = scratch.gpr();
101     GPRReg scratch2GPR = scratch2.gpr();
102
103     ASSERT(vectorLength >= numElements);
104     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
105     
106     JITCompiler::JumpList slowCases;
107
108     size_t size = 0;
109     if (hasIndexingHeader)
110         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
111     size += outOfLineCapacity * sizeof(JSValue);
112
113     m_jit.move(TrustedImmPtr(nullptr), storageGPR);
114
115     if (size) {
116         if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
117             m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
118             
119             m_jit.addPtr(
120                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
121                 storageGPR);
122             
123             if (hasIndexingHeader)
124                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
125         } else
126             slowCases.append(m_jit.jump());
127     }
128
129     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
130     Allocator allocator = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
131     if (allocator) {
132         emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
133         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
134     } else
135         slowCases.append(m_jit.jump());
136
137     // I want a slow path that also loads out the storage pointer, and that's
138     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
139     // of work for a very small piece of functionality. :-/
140     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
141         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
142         structure, vectorLength));
143
144     if (numElements < vectorLength) {
145 #if USE(JSVALUE64)
146         if (hasDouble(structure->indexingType()))
147             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
148         else
149             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
150         for (unsigned i = numElements; i < vectorLength; ++i)
151             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
152 #else
153         EncodedValueDescriptor value;
154         if (hasDouble(structure->indexingType()))
155             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
156         else
157             value.asInt64 = JSValue::encode(JSValue());
158         for (unsigned i = numElements; i < vectorLength; ++i) {
159             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
160             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
161         }
162 #endif
163     }
164     
165     if (hasIndexingHeader)
166         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
167     
168     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
169     
170     m_jit.mutatorFence(*m_jit.vm());
171 }
172
173 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
174 {
175     if (inlineCallFrame && !inlineCallFrame->isVarargs())
176         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
177     else {
178         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
179         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
180         if (!includeThis)
181             m_jit.sub32(TrustedImm32(1), lengthGPR);
182     }
183 }
184
185 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
186 {
187     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
188 }
189
190 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
191 {
192     if (origin.inlineCallFrame) {
193         if (origin.inlineCallFrame->isClosureCall) {
194             m_jit.loadPtr(
195                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
196                 calleeGPR);
197         } else {
198             m_jit.move(
199                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
200                 calleeGPR);
201         }
202     } else
203         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
204 }
205
206 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
207 {
208     m_jit.addPtr(
209         TrustedImm32(
210             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
211         GPRInfo::callFrameRegister, startGPR);
212 }
213
214 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
215 {
216     if (!Options::useOSRExitFuzz()
217         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
218         || !doOSRExitFuzzing())
219         return MacroAssembler::Jump();
220     
221     MacroAssembler::Jump result;
222     
223     m_jit.pushToSave(GPRInfo::regT0);
224     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
225     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
226     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
227     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
228     unsigned at = Options::fireOSRExitFuzzAt();
229     if (at || atOrAfter) {
230         unsigned threshold;
231         MacroAssembler::RelationalCondition condition;
232         if (atOrAfter) {
233             threshold = atOrAfter;
234             condition = MacroAssembler::Below;
235         } else {
236             threshold = at;
237             condition = MacroAssembler::NotEqual;
238         }
239         MacroAssembler::Jump ok = m_jit.branch32(
240             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
241         m_jit.popToRestore(GPRInfo::regT0);
242         result = m_jit.jump();
243         ok.link(&m_jit);
244     }
245     m_jit.popToRestore(GPRInfo::regT0);
246     
247     return result;
248 }
249
250 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
251 {
252     if (!m_compileOkay)
253         return;
254     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
255     if (fuzzJump.isSet()) {
256         JITCompiler::JumpList jumpsToFail;
257         jumpsToFail.append(fuzzJump);
258         jumpsToFail.append(jumpToFail);
259         m_jit.appendExitInfo(jumpsToFail);
260     } else
261         m_jit.appendExitInfo(jumpToFail);
262     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
263 }
264
265 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
266 {
267     if (!m_compileOkay)
268         return;
269     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
270     if (fuzzJump.isSet()) {
271         JITCompiler::JumpList myJumpsToFail;
272         myJumpsToFail.append(jumpsToFail);
273         myJumpsToFail.append(fuzzJump);
274         m_jit.appendExitInfo(myJumpsToFail);
275     } else
276         m_jit.appendExitInfo(jumpsToFail);
277     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
278 }
279
280 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
281 {
282     if (!m_compileOkay)
283         return OSRExitJumpPlaceholder();
284     unsigned index = m_jit.jitCode()->osrExit.size();
285     m_jit.appendExitInfo();
286     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
287     return OSRExitJumpPlaceholder(index);
288 }
289
290 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
291 {
292     return speculationCheck(kind, jsValueSource, nodeUse.node());
293 }
294
295 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
296 {
297     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
298 }
299
300 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
301 {
302     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
303 }
304
305 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
306 {
307     if (!m_compileOkay)
308         return;
309     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
310     m_jit.appendExitInfo(jumpToFail);
311     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
312 }
313
314 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
315 {
316     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
317 }
318
319 void SpeculativeJIT::emitInvalidationPoint(Node* node)
320 {
321     if (!m_compileOkay)
322         return;
323     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
324     m_jit.jitCode()->appendOSRExit(OSRExit(
325         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
326         this, m_stream->size()));
327     info.m_replacementSource = m_jit.watchpointLabel();
328     ASSERT(info.m_replacementSource.isSet());
329     noResult(node);
330 }
331
332 void SpeculativeJIT::unreachable(Node* node)
333 {
334     m_compileOkay = false;
335     m_jit.abortWithReason(DFGUnreachableNode, node->op());
336 }
337
338 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
339 {
340     if (!m_compileOkay)
341         return;
342     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
343     m_compileOkay = false;
344     if (verboseCompilationEnabled())
345         dataLog("Bailing compilation.\n");
346 }
347
348 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
349 {
350     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
351 }
352
353 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
354 {
355     ASSERT(needsTypeCheck(edge, typesPassedThrough));
356     m_interpreter.filter(edge, typesPassedThrough);
357     speculationCheck(exitKind, source, edge.node(), jumpToFail);
358 }
359
360 RegisterSet SpeculativeJIT::usedRegisters()
361 {
362     RegisterSet result;
363     
364     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
365         GPRReg gpr = GPRInfo::toRegister(i);
366         if (m_gprs.isInUse(gpr))
367             result.set(gpr);
368     }
369     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
370         FPRReg fpr = FPRInfo::toRegister(i);
371         if (m_fprs.isInUse(fpr))
372             result.set(fpr);
373     }
374     
375     result.merge(RegisterSet::stubUnavailableRegisters());
376     
377     return result;
378 }
379
380 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
381 {
382     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
383 }
384
385 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
386 {
387     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
388 }
389
390 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
391 {
392     for (auto& slowPathGenerator : m_slowPathGenerators) {
393         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
394         slowPathGenerator->generate(this);
395     }
396     for (auto& slowPathLambda : m_slowPathLambdas) {
397         Node* currentNode = slowPathLambda.currentNode;
398         m_currentNode = currentNode;
399         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
400         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
401         slowPathLambda.generator();
402         m_outOfLineStreamIndex = std::nullopt;
403     }
404 }
405
406 void SpeculativeJIT::clearGenerationInfo()
407 {
408     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
409         m_generationInfo[i] = GenerationInfo();
410     m_gprs = RegisterBank<GPRInfo>();
411     m_fprs = RegisterBank<FPRInfo>();
412 }
413
414 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
415 {
416     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
417     Node* node = info.node();
418     DataFormat registerFormat = info.registerFormat();
419     ASSERT(registerFormat != DataFormatNone);
420     ASSERT(registerFormat != DataFormatDouble);
421         
422     SilentSpillAction spillAction;
423     SilentFillAction fillAction;
424         
425     if (!info.needsSpill())
426         spillAction = DoNothingForSpill;
427     else {
428 #if USE(JSVALUE64)
429         ASSERT(info.gpr() == source);
430         if (registerFormat == DataFormatInt32)
431             spillAction = Store32Payload;
432         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
433             spillAction = StorePtr;
434         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
435             spillAction = Store64;
436         else {
437             ASSERT(registerFormat & DataFormatJS);
438             spillAction = Store64;
439         }
440 #elif USE(JSVALUE32_64)
441         if (registerFormat & DataFormatJS) {
442             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
443             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
444         } else {
445             ASSERT(info.gpr() == source);
446             spillAction = Store32Payload;
447         }
448 #endif
449     }
450         
451     if (registerFormat == DataFormatInt32) {
452         ASSERT(info.gpr() == source);
453         ASSERT(isJSInt32(info.registerFormat()));
454         if (node->hasConstant()) {
455             ASSERT(node->isInt32Constant());
456             fillAction = SetInt32Constant;
457         } else
458             fillAction = Load32Payload;
459     } else if (registerFormat == DataFormatBoolean) {
460 #if USE(JSVALUE64)
461         RELEASE_ASSERT_NOT_REACHED();
462 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
463         fillAction = DoNothingForFill;
464 #endif
465 #elif USE(JSVALUE32_64)
466         ASSERT(info.gpr() == source);
467         if (node->hasConstant()) {
468             ASSERT(node->isBooleanConstant());
469             fillAction = SetBooleanConstant;
470         } else
471             fillAction = Load32Payload;
472 #endif
473     } else if (registerFormat == DataFormatCell) {
474         ASSERT(info.gpr() == source);
475         if (node->hasConstant()) {
476             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
477             node->asCell(); // To get the assertion.
478             fillAction = SetCellConstant;
479         } else {
480 #if USE(JSVALUE64)
481             fillAction = LoadPtr;
482 #else
483             fillAction = Load32Payload;
484 #endif
485         }
486     } else if (registerFormat == DataFormatStorage) {
487         ASSERT(info.gpr() == source);
488         fillAction = LoadPtr;
489     } else if (registerFormat == DataFormatInt52) {
490         if (node->hasConstant())
491             fillAction = SetInt52Constant;
492         else if (info.spillFormat() == DataFormatInt52)
493             fillAction = Load64;
494         else if (info.spillFormat() == DataFormatStrictInt52)
495             fillAction = Load64ShiftInt52Left;
496         else if (info.spillFormat() == DataFormatNone)
497             fillAction = Load64;
498         else {
499             RELEASE_ASSERT_NOT_REACHED();
500 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
501             fillAction = Load64; // Make GCC happy.
502 #endif
503         }
504     } else if (registerFormat == DataFormatStrictInt52) {
505         if (node->hasConstant())
506             fillAction = SetStrictInt52Constant;
507         else if (info.spillFormat() == DataFormatInt52)
508             fillAction = Load64ShiftInt52Right;
509         else if (info.spillFormat() == DataFormatStrictInt52)
510             fillAction = Load64;
511         else if (info.spillFormat() == DataFormatNone)
512             fillAction = Load64;
513         else {
514             RELEASE_ASSERT_NOT_REACHED();
515 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
516             fillAction = Load64; // Make GCC happy.
517 #endif
518         }
519     } else {
520         ASSERT(registerFormat & DataFormatJS);
521 #if USE(JSVALUE64)
522         ASSERT(info.gpr() == source);
523         if (node->hasConstant()) {
524             if (node->isCellConstant())
525                 fillAction = SetTrustedJSConstant;
526             else
527                 fillAction = SetJSConstant;
528         } else if (info.spillFormat() == DataFormatInt32) {
529             ASSERT(registerFormat == DataFormatJSInt32);
530             fillAction = Load32PayloadBoxInt;
531         } else
532             fillAction = Load64;
533 #else
534         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
535         if (node->hasConstant())
536             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
537         else if (info.payloadGPR() == source)
538             fillAction = Load32Payload;
539         else { // Fill the Tag
540             switch (info.spillFormat()) {
541             case DataFormatInt32:
542                 ASSERT(registerFormat == DataFormatJSInt32);
543                 fillAction = SetInt32Tag;
544                 break;
545             case DataFormatCell:
546                 ASSERT(registerFormat == DataFormatJSCell);
547                 fillAction = SetCellTag;
548                 break;
549             case DataFormatBoolean:
550                 ASSERT(registerFormat == DataFormatJSBoolean);
551                 fillAction = SetBooleanTag;
552                 break;
553             default:
554                 fillAction = Load32Tag;
555                 break;
556             }
557         }
558 #endif
559     }
560         
561     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
562 }
563     
564 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
565 {
566     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
567     Node* node = info.node();
568     ASSERT(info.registerFormat() == DataFormatDouble);
569
570     SilentSpillAction spillAction;
571     SilentFillAction fillAction;
572         
573     if (!info.needsSpill())
574         spillAction = DoNothingForSpill;
575     else {
576         ASSERT(!node->hasConstant());
577         ASSERT(info.spillFormat() == DataFormatNone);
578         ASSERT(info.fpr() == source);
579         spillAction = StoreDouble;
580     }
581         
582 #if USE(JSVALUE64)
583     if (node->hasConstant()) {
584         node->asNumber(); // To get the assertion.
585         fillAction = SetDoubleConstant;
586     } else {
587         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
588         fillAction = LoadDouble;
589     }
590 #elif USE(JSVALUE32_64)
591     ASSERT(info.registerFormat() == DataFormatDouble);
592     if (node->hasConstant()) {
593         node->asNumber(); // To get the assertion.
594         fillAction = SetDoubleConstant;
595     } else
596         fillAction = LoadDouble;
597 #endif
598
599     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
600 }
601     
602 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
603 {
604     switch (plan.spillAction()) {
605     case DoNothingForSpill:
606         break;
607     case Store32Tag:
608         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
609         break;
610     case Store32Payload:
611         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
612         break;
613     case StorePtr:
614         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
615         break;
616 #if USE(JSVALUE64)
617     case Store64:
618         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
619         break;
620 #endif
621     case StoreDouble:
622         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
623         break;
624     default:
625         RELEASE_ASSERT_NOT_REACHED();
626     }
627 }
628     
629 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
630 {
631     switch (plan.fillAction()) {
632     case DoNothingForFill:
633         break;
634     case SetInt32Constant:
635         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
636         break;
637 #if USE(JSVALUE64)
638     case SetInt52Constant:
639         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
640         break;
641     case SetStrictInt52Constant:
642         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
643         break;
644 #endif // USE(JSVALUE64)
645     case SetBooleanConstant:
646         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
647         break;
648     case SetCellConstant:
649         ASSERT(plan.node()->constant()->value().isCell());
650         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
651         break;
652 #if USE(JSVALUE64)
653     case SetTrustedJSConstant:
654         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
655         break;
656     case SetJSConstant:
657         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
658         break;
659     case SetDoubleConstant:
660         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
661         break;
662     case Load32PayloadBoxInt:
663         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
664         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
665         break;
666     case Load32PayloadConvertToInt52:
667         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
668         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
669         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
670         break;
671     case Load32PayloadSignExtend:
672         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
673         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
674         break;
675 #else
676     case SetJSConstantTag:
677         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
678         break;
679     case SetJSConstantPayload:
680         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
681         break;
682     case SetInt32Tag:
683         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
684         break;
685     case SetCellTag:
686         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
687         break;
688     case SetBooleanTag:
689         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
690         break;
691     case SetDoubleConstant:
692         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
693         break;
694 #endif
695     case Load32Tag:
696         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
697         break;
698     case Load32Payload:
699         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
700         break;
701     case LoadPtr:
702         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
703         break;
704 #if USE(JSVALUE64)
705     case Load64:
706         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
707         break;
708     case Load64ShiftInt52Right:
709         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
710         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
711         break;
712     case Load64ShiftInt52Left:
713         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
714         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
715         break;
716 #endif
717     case LoadDouble:
718         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
719         break;
720     default:
721         RELEASE_ASSERT_NOT_REACHED();
722     }
723 }
724
725 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
726 {
727     JITCompiler::JumpList result;
728     
729     switch (arrayMode.type()) {
730     case Array::Int32:
731     case Array::Double:
732     case Array::Contiguous:
733     case Array::Undecided:
734     case Array::ArrayStorage: {
735         IndexingType shape = arrayMode.shapeMask();
736         switch (arrayMode.arrayClass()) {
737         case Array::OriginalArray:
738             RELEASE_ASSERT_NOT_REACHED();
739             return result;
740
741         case Array::Array:
742             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
743             result.append(m_jit.branch32(
744                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
745             return result;
746
747         case Array::NonArray:
748         case Array::OriginalNonArray:
749             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
750             result.append(m_jit.branch32(
751                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
752             return result;
753
754         case Array::PossiblyArray:
755             m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
756             result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
757             return result;
758         }
759
760         RELEASE_ASSERT_NOT_REACHED();
761         return result;
762     }
763
764     case Array::SlowPutArrayStorage: {
765         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
766
767         switch (arrayMode.arrayClass()) {
768         case Array::OriginalArray:
769             RELEASE_ASSERT_NOT_REACHED();
770             return result;
771
772         case Array::Array:
773             result.append(
774                 m_jit.branchTest32(
775                     MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
776             break;
777
778         case Array::NonArray:
779         case Array::OriginalNonArray:
780             result.append(
781                 m_jit.branchTest32(
782                     MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
783             break;
784
785         case Array::PossiblyArray:
786             break;
787         }
788
789         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
790         m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
791         result.append(
792             m_jit.branch32(
793                 MacroAssembler::Above, tempGPR,
794                 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
795         return result;
796     }
797     default:
798         CRASH();
799         break;
800     }
801     
802     return result;
803 }
804
805 void SpeculativeJIT::checkArray(Node* node)
806 {
807     ASSERT(node->arrayMode().isSpecific());
808     ASSERT(!node->arrayMode().doesConversion());
809     
810     SpeculateCellOperand base(this, node->child1());
811     GPRReg baseReg = base.gpr();
812     
813     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
814         noResult(m_currentNode);
815         return;
816     }
817     
818     switch (node->arrayMode().type()) {
819     case Array::AnyTypedArray:
820     case Array::String:
821         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
822         return;
823     case Array::Int32:
824     case Array::Double:
825     case Array::Contiguous:
826     case Array::Undecided:
827     case Array::ArrayStorage:
828     case Array::SlowPutArrayStorage: {
829         GPRTemporary temp(this);
830         GPRReg tempGPR = temp.gpr();
831         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
832         speculationCheck(
833             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
834             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
835         
836         noResult(m_currentNode);
837         return;
838     }
839     case Array::DirectArguments:
840         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
841         noResult(m_currentNode);
842         return;
843     case Array::ScopedArguments:
844         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
845         noResult(m_currentNode);
846         return;
847     default:
848         speculateCellTypeWithoutTypeFiltering(
849             node->child1(), baseReg,
850             typeForTypedArrayType(node->arrayMode().typedArrayType()));
851         noResult(m_currentNode);
852         return;
853     }
854 }
855
856 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
857 {
858     ASSERT(node->arrayMode().doesConversion());
859     
860     GPRTemporary temp(this);
861     GPRTemporary structure;
862     GPRReg tempGPR = temp.gpr();
863     GPRReg structureGPR = InvalidGPRReg;
864     
865     if (node->op() != ArrayifyToStructure) {
866         GPRTemporary realStructure(this);
867         structure.adopt(realStructure);
868         structureGPR = structure.gpr();
869     }
870         
871     // We can skip all that comes next if we already have array storage.
872     MacroAssembler::JumpList slowPath;
873     
874     if (node->op() == ArrayifyToStructure) {
875         slowPath.append(m_jit.branchWeakStructure(
876             JITCompiler::NotEqual,
877             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
878             node->structure()));
879     } else {
880         m_jit.load8(
881             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
882         
883         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
884     }
885     
886     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
887         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
888     
889     noResult(m_currentNode);
890 }
891
892 void SpeculativeJIT::arrayify(Node* node)
893 {
894     ASSERT(node->arrayMode().isSpecific());
895     
896     SpeculateCellOperand base(this, node->child1());
897     
898     if (!node->child2()) {
899         arrayify(node, base.gpr(), InvalidGPRReg);
900         return;
901     }
902     
903     SpeculateInt32Operand property(this, node->child2());
904     
905     arrayify(node, base.gpr(), property.gpr());
906 }
907
908 GPRReg SpeculativeJIT::fillStorage(Edge edge)
909 {
910     VirtualRegister virtualRegister = edge->virtualRegister();
911     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
912     
913     switch (info.registerFormat()) {
914     case DataFormatNone: {
915         if (info.spillFormat() == DataFormatStorage) {
916             GPRReg gpr = allocate();
917             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
918             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
919             info.fillStorage(*m_stream, gpr);
920             return gpr;
921         }
922         
923         // Must be a cell; fill it as a cell and then return the pointer.
924         return fillSpeculateCell(edge);
925     }
926         
927     case DataFormatStorage: {
928         GPRReg gpr = info.gpr();
929         m_gprs.lock(gpr);
930         return gpr;
931     }
932         
933     default:
934         return fillSpeculateCell(edge);
935     }
936 }
937
938 void SpeculativeJIT::useChildren(Node* node)
939 {
940     if (node->flags() & NodeHasVarArgs) {
941         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
942             if (!!m_jit.graph().m_varArgChildren[childIdx])
943                 use(m_jit.graph().m_varArgChildren[childIdx]);
944         }
945     } else {
946         Edge child1 = node->child1();
947         if (!child1) {
948             ASSERT(!node->child2() && !node->child3());
949             return;
950         }
951         use(child1);
952         
953         Edge child2 = node->child2();
954         if (!child2) {
955             ASSERT(!node->child3());
956             return;
957         }
958         use(child2);
959         
960         Edge child3 = node->child3();
961         if (!child3)
962             return;
963         use(child3);
964     }
965 }
966
967 void SpeculativeJIT::compileTryGetById(Node* node)
968 {
969     switch (node->child1().useKind()) {
970     case CellUse: {
971         SpeculateCellOperand base(this, node->child1());
972         JSValueRegsTemporary result(this, Reuse, base);
973
974         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
975         JSValueRegs resultRegs = result.regs();
976
977         base.use();
978
979         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, AccessType::TryGet);
980
981         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
982         break;
983     }
984
985     case UntypedUse: {
986         JSValueOperand base(this, node->child1());
987         JSValueRegsTemporary result(this, Reuse, base);
988
989         JSValueRegs baseRegs = base.jsValueRegs();
990         JSValueRegs resultRegs = result.regs();
991
992         base.use();
993
994         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
995
996         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, AccessType::TryGet);
997
998         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
999         break;
1000     }
1001
1002     default:
1003         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1004         break;
1005     } 
1006 }
1007
1008 void SpeculativeJIT::compileIn(Node* node)
1009 {
1010     SpeculateCellOperand base(this, node->child1());
1011     GPRReg baseGPR = base.gpr();
1012     
1013     if (JSString* string = node->child2()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
1014         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1015             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1016             
1017             GPRTemporary result(this);
1018             GPRReg resultGPR = result.gpr();
1019
1020             use(node->child2());
1021             
1022             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1023             MacroAssembler::Label done = m_jit.label();
1024             
1025             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1026             // we can cast it to const AtomicStringImpl* safely.
1027             auto slowPath = slowPathCall(
1028                 jump.m_jump, this, operationInOptimize,
1029                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1030                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1031             
1032             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1033             stubInfo->codeOrigin = node->origin.semantic;
1034             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1035             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1036             stubInfo->patch.thisGPR = static_cast<int8_t>(InvalidGPRReg);
1037 #if USE(JSVALUE32_64)
1038             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1039             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1040             stubInfo->patch.thisTagGPR = static_cast<int8_t>(InvalidGPRReg);
1041 #endif
1042             stubInfo->patch.usedRegisters = usedRegisters();
1043
1044             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1045             addSlowPathGenerator(WTFMove(slowPath));
1046
1047             base.use();
1048
1049             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1050             return;
1051         }
1052     }
1053
1054     JSValueOperand key(this, node->child2());
1055     JSValueRegs regs = key.jsValueRegs();
1056         
1057     GPRFlushedCallResult result(this);
1058     GPRReg resultGPR = result.gpr();
1059         
1060     base.use();
1061     key.use();
1062         
1063     flushRegisters();
1064     callOperation(
1065         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1066         baseGPR, regs);
1067     m_jit.exceptionCheck();
1068     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1069 }
1070
1071 void SpeculativeJIT::compileDeleteById(Node* node)
1072 {
1073     JSValueOperand value(this, node->child1());
1074     GPRFlushedCallResult result(this);
1075
1076     JSValueRegs valueRegs = value.jsValueRegs();
1077     GPRReg resultGPR = result.gpr();
1078
1079     value.use();
1080
1081     flushRegisters();
1082     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1083     m_jit.exceptionCheck();
1084
1085     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1086 }
1087
1088 void SpeculativeJIT::compileDeleteByVal(Node* node)
1089 {
1090     JSValueOperand base(this, node->child1());
1091     JSValueOperand key(this, node->child2());
1092     GPRFlushedCallResult result(this);
1093
1094     JSValueRegs baseRegs = base.jsValueRegs();
1095     JSValueRegs keyRegs = key.jsValueRegs();
1096     GPRReg resultGPR = result.gpr();
1097
1098     base.use();
1099     key.use();
1100
1101     flushRegisters();
1102     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1103     m_jit.exceptionCheck();
1104
1105     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1106 }
1107
1108 void SpeculativeJIT::compilePushWithScope(Node* node)
1109 {
1110     SpeculateCellOperand currentScope(this, node->child1());
1111     GPRReg currentScopeGPR = currentScope.gpr();
1112
1113     GPRFlushedCallResult result(this);
1114     GPRReg resultGPR = result.gpr();
1115
1116     auto objectEdge = node->child2();
1117     if (objectEdge.useKind() == ObjectUse) {
1118         SpeculateCellOperand object(this, objectEdge);
1119         GPRReg objectGPR = object.gpr();
1120         speculateObject(objectEdge, objectGPR);
1121
1122         flushRegisters();
1123         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1124         // No exception check here as we did not have to call toObject().
1125     } else {
1126         ASSERT(objectEdge.useKind() == UntypedUse);
1127         JSValueOperand object(this, objectEdge);
1128         JSValueRegs objectRegs = object.jsValueRegs();
1129
1130         flushRegisters();
1131         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1132         m_jit.exceptionCheck();
1133     }
1134     
1135     cellResult(resultGPR, node);
1136 }
1137
1138 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1139 {
1140     unsigned branchIndexInBlock = detectPeepHoleBranch();
1141     if (branchIndexInBlock != UINT_MAX) {
1142         Node* branchNode = m_block->at(branchIndexInBlock);
1143
1144         ASSERT(node->adjustedRefCount() == 1);
1145         
1146         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1147     
1148         m_indexInBlock = branchIndexInBlock;
1149         m_currentNode = branchNode;
1150         
1151         return true;
1152     }
1153     
1154     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1155     
1156     return false;
1157 }
1158
1159 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1160 {
1161     unsigned branchIndexInBlock = detectPeepHoleBranch();
1162     if (branchIndexInBlock != UINT_MAX) {
1163         Node* branchNode = m_block->at(branchIndexInBlock);
1164
1165         ASSERT(node->adjustedRefCount() == 1);
1166         
1167         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1168     
1169         m_indexInBlock = branchIndexInBlock;
1170         m_currentNode = branchNode;
1171         
1172         return true;
1173     }
1174     
1175     nonSpeculativeNonPeepholeStrictEq(node, invert);
1176     
1177     return false;
1178 }
1179
1180 static const char* dataFormatString(DataFormat format)
1181 {
1182     // These values correspond to the DataFormat enum.
1183     const char* strings[] = {
1184         "[  ]",
1185         "[ i]",
1186         "[ d]",
1187         "[ c]",
1188         "Err!",
1189         "Err!",
1190         "Err!",
1191         "Err!",
1192         "[J ]",
1193         "[Ji]",
1194         "[Jd]",
1195         "[Jc]",
1196         "Err!",
1197         "Err!",
1198         "Err!",
1199         "Err!",
1200     };
1201     return strings[format];
1202 }
1203
1204 void SpeculativeJIT::dump(const char* label)
1205 {
1206     if (label)
1207         dataLogF("<%s>\n", label);
1208
1209     dataLogF("  gprs:\n");
1210     m_gprs.dump();
1211     dataLogF("  fprs:\n");
1212     m_fprs.dump();
1213     dataLogF("  VirtualRegisters:\n");
1214     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1215         GenerationInfo& info = m_generationInfo[i];
1216         if (info.alive())
1217             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1218         else
1219             dataLogF("    % 3d:[__][__]", i);
1220         if (info.registerFormat() == DataFormatDouble)
1221             dataLogF(":fpr%d\n", info.fpr());
1222         else if (info.registerFormat() != DataFormatNone
1223 #if USE(JSVALUE32_64)
1224             && !(info.registerFormat() & DataFormatJS)
1225 #endif
1226             ) {
1227             ASSERT(info.gpr() != InvalidGPRReg);
1228             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1229         } else
1230             dataLogF("\n");
1231     }
1232     if (label)
1233         dataLogF("</%s>\n", label);
1234 }
1235
1236 GPRTemporary::GPRTemporary()
1237     : m_jit(0)
1238     , m_gpr(InvalidGPRReg)
1239 {
1240 }
1241
1242 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1243     : m_jit(jit)
1244     , m_gpr(InvalidGPRReg)
1245 {
1246     m_gpr = m_jit->allocate();
1247 }
1248
1249 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1250     : m_jit(jit)
1251     , m_gpr(InvalidGPRReg)
1252 {
1253     m_gpr = m_jit->allocate(specific);
1254 }
1255
1256 #if USE(JSVALUE32_64)
1257 GPRTemporary::GPRTemporary(
1258     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1259     : m_jit(jit)
1260     , m_gpr(InvalidGPRReg)
1261 {
1262     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1263         m_gpr = m_jit->reuse(op1.gpr(which));
1264     else
1265         m_gpr = m_jit->allocate();
1266 }
1267 #endif // USE(JSVALUE32_64)
1268
1269 JSValueRegsTemporary::JSValueRegsTemporary() { }
1270
1271 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1272 #if USE(JSVALUE64)
1273     : m_gpr(jit)
1274 #else
1275     : m_payloadGPR(jit)
1276     , m_tagGPR(jit)
1277 #endif
1278 {
1279 }
1280
1281 #if USE(JSVALUE64)
1282 template<typename T>
1283 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1284     : m_gpr(jit, Reuse, operand)
1285 {
1286 }
1287 #else
1288 template<typename T>
1289 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1290 {
1291     if (resultWord == PayloadWord) {
1292         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1293         m_tagGPR = GPRTemporary(jit);
1294     } else {
1295         m_payloadGPR = GPRTemporary(jit);
1296         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1297     }
1298 }
1299 #endif
1300
1301 #if USE(JSVALUE64)
1302 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1303 {
1304     m_gpr = GPRTemporary(jit, Reuse, operand);
1305 }
1306 #else
1307 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1308 {
1309     if (jit->canReuse(operand.node())) {
1310         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1311         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1312     } else {
1313         m_payloadGPR = GPRTemporary(jit);
1314         m_tagGPR = GPRTemporary(jit);
1315     }
1316 }
1317 #endif
1318
1319 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1320
1321 JSValueRegs JSValueRegsTemporary::regs()
1322 {
1323 #if USE(JSVALUE64)
1324     return JSValueRegs(m_gpr.gpr());
1325 #else
1326     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1327 #endif
1328 }
1329
1330 void GPRTemporary::adopt(GPRTemporary& other)
1331 {
1332     ASSERT(!m_jit);
1333     ASSERT(m_gpr == InvalidGPRReg);
1334     ASSERT(other.m_jit);
1335     ASSERT(other.m_gpr != InvalidGPRReg);
1336     m_jit = other.m_jit;
1337     m_gpr = other.m_gpr;
1338     other.m_jit = 0;
1339     other.m_gpr = InvalidGPRReg;
1340 }
1341
1342 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1343 {
1344     ASSERT(other.m_jit);
1345     ASSERT(other.m_fpr != InvalidFPRReg);
1346     m_jit = other.m_jit;
1347     m_fpr = other.m_fpr;
1348
1349     other.m_jit = nullptr;
1350 }
1351
1352 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1353     : m_jit(jit)
1354     , m_fpr(InvalidFPRReg)
1355 {
1356     m_fpr = m_jit->fprAllocate();
1357 }
1358
1359 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1360     : m_jit(jit)
1361     , m_fpr(InvalidFPRReg)
1362 {
1363     if (m_jit->canReuse(op1.node()))
1364         m_fpr = m_jit->reuse(op1.fpr());
1365     else
1366         m_fpr = m_jit->fprAllocate();
1367 }
1368
1369 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1370     : m_jit(jit)
1371     , m_fpr(InvalidFPRReg)
1372 {
1373     if (m_jit->canReuse(op1.node()))
1374         m_fpr = m_jit->reuse(op1.fpr());
1375     else if (m_jit->canReuse(op2.node()))
1376         m_fpr = m_jit->reuse(op2.fpr());
1377     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1378         m_fpr = m_jit->reuse(op1.fpr());
1379     else
1380         m_fpr = m_jit->fprAllocate();
1381 }
1382
1383 #if USE(JSVALUE32_64)
1384 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1385     : m_jit(jit)
1386     , m_fpr(InvalidFPRReg)
1387 {
1388     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1389         m_fpr = m_jit->reuse(op1.fpr());
1390     else
1391         m_fpr = m_jit->fprAllocate();
1392 }
1393 #endif
1394
1395 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1396 {
1397     BasicBlock* taken = branchNode->branchData()->taken.block;
1398     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1399
1400     if (taken == nextBlock()) {
1401         condition = MacroAssembler::invert(condition);
1402         std::swap(taken, notTaken);
1403     }
1404
1405     SpeculateDoubleOperand op1(this, node->child1());
1406     SpeculateDoubleOperand op2(this, node->child2());
1407     
1408     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1409     jump(notTaken);
1410 }
1411
1412 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1413 {
1414     BasicBlock* taken = branchNode->branchData()->taken.block;
1415     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1416
1417     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1418     
1419     if (taken == nextBlock()) {
1420         condition = MacroAssembler::NotEqual;
1421         BasicBlock* tmp = taken;
1422         taken = notTaken;
1423         notTaken = tmp;
1424     }
1425
1426     SpeculateCellOperand op1(this, node->child1());
1427     SpeculateCellOperand op2(this, node->child2());
1428     
1429     GPRReg op1GPR = op1.gpr();
1430     GPRReg op2GPR = op2.gpr();
1431     
1432     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1433         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1434             speculationCheck(
1435                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1436         }
1437         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1438             speculationCheck(
1439                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1440         }
1441     } else {
1442         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1443             speculationCheck(
1444                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1445                 m_jit.branchIfNotObject(op1GPR));
1446         }
1447         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1448             m_jit.branchTest8(
1449                 MacroAssembler::NonZero, 
1450                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1451                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1452
1453         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1454             speculationCheck(
1455                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1456                 m_jit.branchIfNotObject(op2GPR));
1457         }
1458         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1459             m_jit.branchTest8(
1460                 MacroAssembler::NonZero, 
1461                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1462                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1463     }
1464
1465     branchPtr(condition, op1GPR, op2GPR, taken);
1466     jump(notTaken);
1467 }
1468
1469 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1470 {
1471     BasicBlock* taken = branchNode->branchData()->taken.block;
1472     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1473
1474     // The branch instruction will branch to the taken block.
1475     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1476     if (taken == nextBlock()) {
1477         condition = JITCompiler::invert(condition);
1478         BasicBlock* tmp = taken;
1479         taken = notTaken;
1480         notTaken = tmp;
1481     }
1482
1483     if (node->child1()->isInt32Constant()) {
1484         int32_t imm = node->child1()->asInt32();
1485         SpeculateBooleanOperand op2(this, node->child2());
1486         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1487     } else if (node->child2()->isInt32Constant()) {
1488         SpeculateBooleanOperand op1(this, node->child1());
1489         int32_t imm = node->child2()->asInt32();
1490         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1491     } else {
1492         SpeculateBooleanOperand op1(this, node->child1());
1493         SpeculateBooleanOperand op2(this, node->child2());
1494         branch32(condition, op1.gpr(), op2.gpr(), taken);
1495     }
1496
1497     jump(notTaken);
1498 }
1499
1500 void SpeculativeJIT::compileStringSlice(Node* node)
1501 {
1502     SpeculateCellOperand string(this, node->child1());
1503     GPRTemporary startIndex(this);
1504     GPRTemporary temp(this);
1505     GPRTemporary temp2(this);
1506
1507     GPRReg stringGPR = string.gpr();
1508     GPRReg startIndexGPR = startIndex.gpr();
1509     GPRReg tempGPR = temp.gpr();
1510     GPRReg temp2GPR = temp2.gpr();
1511
1512     speculateString(node->child1(), stringGPR);
1513
1514     {
1515         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1516
1517         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1518         if (node->child3())
1519             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1520         else
1521             m_jit.move(temp2GPR, tempGPR);
1522     }
1523
1524     CCallHelpers::JumpList doneCases;
1525     CCallHelpers::JumpList slowCases;
1526
1527     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1528     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1529     doneCases.append(m_jit.jump());
1530
1531     nonEmptyCase.link(&m_jit);
1532     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1533     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1534
1535     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1536     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1537
1538     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1539
1540     // Load the character into scratchReg
1541     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1542     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1543
1544     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1545     auto cont8Bit = m_jit.jump();
1546
1547     is16Bit.link(&m_jit);
1548     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1549
1550     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1551
1552     // 8 bit string values don't need the isASCII check.
1553     cont8Bit.link(&m_jit);
1554
1555     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1556     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1557     m_jit.loadPtr(tempGPR, tempGPR);
1558
1559     addSlowPathGenerator(
1560         slowPathCall(
1561             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1562
1563     addSlowPathGenerator(
1564         slowPathCall(
1565             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1566
1567     doneCases.link(&m_jit);
1568     cellResult(tempGPR, node);
1569 }
1570
1571 void SpeculativeJIT::compileToLowerCase(Node* node)
1572 {
1573     ASSERT(node->op() == ToLowerCase);
1574     SpeculateCellOperand string(this, node->child1());
1575     GPRTemporary temp(this);
1576     GPRTemporary index(this);
1577     GPRTemporary charReg(this);
1578     GPRTemporary length(this);
1579
1580     GPRReg stringGPR = string.gpr();
1581     GPRReg tempGPR = temp.gpr();
1582     GPRReg indexGPR = index.gpr();
1583     GPRReg charGPR = charReg.gpr();
1584     GPRReg lengthGPR = length.gpr();
1585
1586     speculateString(node->child1(), stringGPR);
1587
1588     CCallHelpers::JumpList slowPath;
1589
1590     m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1591
1592     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1593     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1594
1595     slowPath.append(m_jit.branchTest32(
1596         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1597         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1598     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1599     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1600
1601     auto loopStart = m_jit.label();
1602     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1603     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1604     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1605     m_jit.sub32(TrustedImm32('A'), charGPR);
1606     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1607
1608     m_jit.add32(TrustedImm32(1), indexGPR);
1609     m_jit.jump().linkTo(loopStart, &m_jit);
1610     
1611     slowPath.link(&m_jit);
1612     silentSpillAllRegisters(lengthGPR);
1613     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1614     silentFillAllRegisters();
1615     m_jit.exceptionCheck();
1616     auto done = m_jit.jump();
1617
1618     loopDone.link(&m_jit);
1619     m_jit.move(stringGPR, lengthGPR);
1620
1621     done.link(&m_jit);
1622     cellResult(lengthGPR, node);
1623 }
1624
1625 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1626 {
1627     BasicBlock* taken = branchNode->branchData()->taken.block;
1628     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1629
1630     // The branch instruction will branch to the taken block.
1631     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1632     if (taken == nextBlock()) {
1633         condition = JITCompiler::invert(condition);
1634         BasicBlock* tmp = taken;
1635         taken = notTaken;
1636         notTaken = tmp;
1637     }
1638
1639     if (node->child1()->isInt32Constant()) {
1640         int32_t imm = node->child1()->asInt32();
1641         SpeculateInt32Operand op2(this, node->child2());
1642         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1643     } else if (node->child2()->isInt32Constant()) {
1644         SpeculateInt32Operand op1(this, node->child1());
1645         int32_t imm = node->child2()->asInt32();
1646         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1647     } else {
1648         SpeculateInt32Operand op1(this, node->child1());
1649         SpeculateInt32Operand op2(this, node->child2());
1650         branch32(condition, op1.gpr(), op2.gpr(), taken);
1651     }
1652
1653     jump(notTaken);
1654 }
1655
1656 // Returns true if the compare is fused with a subsequent branch.
1657 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1658 {
1659     // Fused compare & branch.
1660     unsigned branchIndexInBlock = detectPeepHoleBranch();
1661     if (branchIndexInBlock != UINT_MAX) {
1662         Node* branchNode = m_block->at(branchIndexInBlock);
1663
1664         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1665         // so can be no intervening nodes to also reference the compare. 
1666         ASSERT(node->adjustedRefCount() == 1);
1667
1668         if (node->isBinaryUseKind(Int32Use))
1669             compilePeepHoleInt32Branch(node, branchNode, condition);
1670 #if USE(JSVALUE64)
1671         else if (node->isBinaryUseKind(Int52RepUse))
1672             compilePeepHoleInt52Branch(node, branchNode, condition);
1673 #endif // USE(JSVALUE64)
1674         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1675             // Use non-peephole comparison, for now.
1676             return false;
1677         } else if (node->isBinaryUseKind(DoubleRepUse))
1678             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1679         else if (node->op() == CompareEq) {
1680             if (node->isBinaryUseKind(BooleanUse))
1681                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1682             else if (node->isBinaryUseKind(SymbolUse))
1683                 compilePeepHoleSymbolEquality(node, branchNode);
1684             else if (node->isBinaryUseKind(ObjectUse))
1685                 compilePeepHoleObjectEquality(node, branchNode);
1686             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1687                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1688             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1689                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1690             else if (!needsTypeCheck(node->child1(), SpecOther))
1691                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1692             else if (!needsTypeCheck(node->child2(), SpecOther))
1693                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1694             else {
1695                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1696                 return true;
1697             }
1698         } else {
1699             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1700             return true;
1701         }
1702
1703         use(node->child1());
1704         use(node->child2());
1705         m_indexInBlock = branchIndexInBlock;
1706         m_currentNode = branchNode;
1707         return true;
1708     }
1709     return false;
1710 }
1711
1712 void SpeculativeJIT::noticeOSRBirth(Node* node)
1713 {
1714     if (!node->hasVirtualRegister())
1715         return;
1716     
1717     VirtualRegister virtualRegister = node->virtualRegister();
1718     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1719     
1720     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1721 }
1722
1723 void SpeculativeJIT::compileMovHint(Node* node)
1724 {
1725     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1726     
1727     Node* child = node->child1().node();
1728     noticeOSRBirth(child);
1729     
1730     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1731 }
1732
1733 void SpeculativeJIT::bail(AbortReason reason)
1734 {
1735     if (verboseCompilationEnabled())
1736         dataLog("Bailing compilation.\n");
1737     m_compileOkay = true;
1738     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1739     clearGenerationInfo();
1740 }
1741
1742 void SpeculativeJIT::compileCurrentBlock()
1743 {
1744     ASSERT(m_compileOkay);
1745     
1746     if (!m_block)
1747         return;
1748     
1749     ASSERT(m_block->isReachable);
1750     
1751     m_jit.blockHeads()[m_block->index] = m_jit.label();
1752
1753     if (!m_block->intersectionOfCFAHasVisited) {
1754         // Don't generate code for basic blocks that are unreachable according to CFA.
1755         // But to be sure that nobody has generated a jump to this block, drop in a
1756         // breakpoint here.
1757         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1758         return;
1759     }
1760
1761     if (m_block->isCatchEntrypoint) {
1762         m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1763         if (Options::zeroStackFrame())
1764             m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1765         m_jit.emitSaveCalleeSaves();
1766         m_jit.emitMaterializeTagCheckRegisters();
1767         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1768     }
1769
1770     m_stream->appendAndLog(VariableEvent::reset());
1771     
1772     m_jit.jitAssertHasValidCallFrame();
1773     m_jit.jitAssertTagsInPlace();
1774     m_jit.jitAssertArgumentCountSane();
1775
1776     m_state.reset();
1777     m_state.beginBasicBlock(m_block);
1778     
1779     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1780         int operand = m_block->variablesAtHead.operandForIndex(i);
1781         Node* node = m_block->variablesAtHead[i];
1782         if (!node)
1783             continue; // No need to record dead SetLocal's.
1784         
1785         VariableAccessData* variable = node->variableAccessData();
1786         DataFormat format;
1787         if (!node->refCount())
1788             continue; // No need to record dead SetLocal's.
1789         format = dataFormatFor(variable->flushFormat());
1790         m_stream->appendAndLog(
1791             VariableEvent::setLocal(
1792                 VirtualRegister(operand),
1793                 variable->machineLocal(),
1794                 format));
1795     }
1796
1797     m_origin = NodeOrigin();
1798     
1799     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1800         m_currentNode = m_block->at(m_indexInBlock);
1801         
1802         // We may have hit a contradiction that the CFA was aware of but that the JIT
1803         // didn't cause directly.
1804         if (!m_state.isValid()) {
1805             bail(DFGBailedAtTopOfBlock);
1806             return;
1807         }
1808
1809         m_interpreter.startExecuting();
1810         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1811         m_jit.setForNode(m_currentNode);
1812         m_origin = m_currentNode->origin;
1813         if (validationEnabled())
1814             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1815         m_lastGeneratedNode = m_currentNode->op();
1816         
1817         ASSERT(m_currentNode->shouldGenerate());
1818         
1819         if (verboseCompilationEnabled()) {
1820             dataLogF(
1821                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1822                 (int)m_currentNode->index(),
1823                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1824             dataLog("\n");
1825         }
1826
1827         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1828             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1829
1830         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1831
1832         compile(m_currentNode);
1833         
1834         if (belongsInMinifiedGraph(m_currentNode->op()))
1835             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1836         
1837 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1838         m_jit.clearRegisterAllocationOffsets();
1839 #endif
1840         
1841         if (!m_compileOkay) {
1842             bail(DFGBailedAtEndOfNode);
1843             return;
1844         }
1845         
1846         // Make sure that the abstract state is rematerialized for the next node.
1847         m_interpreter.executeEffects(m_indexInBlock);
1848     }
1849     
1850     // Perform the most basic verification that children have been used correctly.
1851     if (!ASSERT_DISABLED) {
1852         for (auto& info : m_generationInfo)
1853             RELEASE_ASSERT(!info.alive());
1854     }
1855 }
1856
1857 // If we are making type predictions about our arguments then
1858 // we need to check that they are correct on function entry.
1859 void SpeculativeJIT::checkArgumentTypes()
1860 {
1861     ASSERT(!m_currentNode);
1862     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1863
1864     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1865     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1866         Node* node = arguments[i];
1867         if (!node) {
1868             // The argument is dead. We don't do any checks for such arguments.
1869             continue;
1870         }
1871         
1872         ASSERT(node->op() == SetArgument);
1873         ASSERT(node->shouldGenerate());
1874
1875         VariableAccessData* variableAccessData = node->variableAccessData();
1876         FlushFormat format = variableAccessData->flushFormat();
1877         
1878         if (format == FlushedJSValue)
1879             continue;
1880         
1881         VirtualRegister virtualRegister = variableAccessData->local();
1882
1883         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1884         
1885 #if USE(JSVALUE64)
1886         switch (format) {
1887         case FlushedInt32: {
1888             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1889             break;
1890         }
1891         case FlushedBoolean: {
1892             GPRTemporary temp(this);
1893             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1894             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1895             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1896             break;
1897         }
1898         case FlushedCell: {
1899             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1900             break;
1901         }
1902         default:
1903             RELEASE_ASSERT_NOT_REACHED();
1904             break;
1905         }
1906 #else
1907         switch (format) {
1908         case FlushedInt32: {
1909             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1910             break;
1911         }
1912         case FlushedBoolean: {
1913             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1914             break;
1915         }
1916         case FlushedCell: {
1917             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1918             break;
1919         }
1920         default:
1921             RELEASE_ASSERT_NOT_REACHED();
1922             break;
1923         }
1924 #endif
1925     }
1926
1927     m_origin = NodeOrigin();
1928 }
1929
1930 bool SpeculativeJIT::compile()
1931 {
1932     checkArgumentTypes();
1933     
1934     ASSERT(!m_currentNode);
1935     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1936         m_jit.setForBlockIndex(blockIndex);
1937         m_block = m_jit.graph().block(blockIndex);
1938         compileCurrentBlock();
1939     }
1940     linkBranches();
1941     return true;
1942 }
1943
1944 void SpeculativeJIT::createOSREntries()
1945 {
1946     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1947         BasicBlock* block = m_jit.graph().block(blockIndex);
1948         if (!block)
1949             continue;
1950         if (block->isOSRTarget || block->isCatchEntrypoint) {
1951             // Currently we don't have OSR entry trampolines. We could add them
1952             // here if need be.
1953             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1954         }
1955     }
1956 }
1957
1958 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1959 {
1960     unsigned osrEntryIndex = 0;
1961     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1962         BasicBlock* block = m_jit.graph().block(blockIndex);
1963         if (!block)
1964             continue;
1965         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1966             continue;
1967         if (block->isCatchEntrypoint) {
1968             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1969             Vector<FlushFormat> argumentFormats;
1970             argumentFormats.reserveInitialCapacity(argumentsVector.size());
1971             for (Node* setArgument : argumentsVector) {
1972                 if (setArgument) {
1973                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
1974                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
1975                     argumentFormats.uncheckedAppend(flushFormat);
1976                 } else
1977                     argumentFormats.uncheckedAppend(DeadFlush);
1978             }
1979             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
1980         } else {
1981             ASSERT(block->isOSRTarget);
1982             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
1983         }
1984     }
1985
1986     m_jit.jitCode()->finalizeOSREntrypoints();
1987     m_jit.jitCode()->common.finalizeCatchEntrypoints();
1988
1989     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
1990     
1991     if (verboseCompilationEnabled()) {
1992         DumpContext dumpContext;
1993         dataLog("OSR Entries:\n");
1994         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
1995             dataLog("    ", inContext(entryData, &dumpContext), "\n");
1996         if (!dumpContext.isEmpty())
1997             dumpContext.dump(WTF::dataFile());
1998     }
1999 }
2000     
2001 void SpeculativeJIT::compileCheckTraps(Node*)
2002 {
2003     ASSERT(Options::usePollingTraps());
2004     GPRTemporary unused(this);
2005     GPRReg unusedGPR = unused.gpr();
2006
2007     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2008         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2009
2010     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2011 }
2012
2013 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2014 {
2015     Edge child3 = m_jit.graph().varArgChild(node, 2);
2016     Edge child4 = m_jit.graph().varArgChild(node, 3);
2017
2018     ArrayMode arrayMode = node->arrayMode();
2019     
2020     GPRReg baseReg = base.gpr();
2021     GPRReg propertyReg = property.gpr();
2022     
2023     SpeculateDoubleOperand value(this, child3);
2024
2025     FPRReg valueReg = value.fpr();
2026     
2027     DFG_TYPE_CHECK(
2028         JSValueRegs(), child3, SpecFullRealNumber,
2029         m_jit.branchDouble(
2030             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2031     
2032     if (!m_compileOkay)
2033         return;
2034     
2035     StorageOperand storage(this, child4);
2036     GPRReg storageReg = storage.gpr();
2037
2038     if (node->op() == PutByValAlias) {
2039         // Store the value to the array.
2040         GPRReg propertyReg = property.gpr();
2041         FPRReg valueReg = value.fpr();
2042         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2043         
2044         noResult(m_currentNode);
2045         return;
2046     }
2047     
2048     GPRTemporary temporary;
2049     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2050
2051     MacroAssembler::Jump slowCase;
2052     
2053     if (arrayMode.isInBounds()) {
2054         speculationCheck(
2055             OutOfBounds, JSValueRegs(), 0,
2056             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2057     } else {
2058         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2059         
2060         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2061         
2062         if (!arrayMode.isOutOfBounds())
2063             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2064         
2065         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2066         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2067
2068         inBounds.link(&m_jit);
2069     }
2070     
2071     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2072
2073     base.use();
2074     property.use();
2075     value.use();
2076     storage.use();
2077     
2078     if (arrayMode.isOutOfBounds()) {
2079         addSlowPathGenerator(
2080             slowPathCall(
2081                 slowCase, this,
2082                 m_jit.codeBlock()->isStrictMode()
2083                     ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2084                     : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2085                 NoResult, baseReg, propertyReg, valueReg));
2086     }
2087
2088     noResult(m_currentNode, UseChildrenCalledExplicitly);
2089 }
2090
2091 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2092 {
2093     SpeculateCellOperand string(this, node->child1());
2094     SpeculateStrictInt32Operand index(this, node->child2());
2095     StorageOperand storage(this, node->child3());
2096
2097     GPRReg stringReg = string.gpr();
2098     GPRReg indexReg = index.gpr();
2099     GPRReg storageReg = storage.gpr();
2100     
2101     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2102
2103     // unsigned comparison so we can filter out negative indices and indices that are too large
2104     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2105
2106     GPRTemporary scratch(this);
2107     GPRReg scratchReg = scratch.gpr();
2108
2109     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2110
2111     // Load the character into scratchReg
2112     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2113
2114     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2115     JITCompiler::Jump cont8Bit = m_jit.jump();
2116
2117     is16Bit.link(&m_jit);
2118
2119     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2120
2121     cont8Bit.link(&m_jit);
2122
2123     int32Result(scratchReg, m_currentNode);
2124 }
2125
2126 void SpeculativeJIT::compileGetByValOnString(Node* node)
2127 {
2128     SpeculateCellOperand base(this, m_graph.child(node, 0));
2129     SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2130     StorageOperand storage(this, m_graph.child(node, 2));
2131     GPRReg baseReg = base.gpr();
2132     GPRReg propertyReg = property.gpr();
2133     GPRReg storageReg = storage.gpr();
2134
2135     GPRTemporary scratch(this);
2136     GPRReg scratchReg = scratch.gpr();
2137 #if USE(JSVALUE32_64)
2138     GPRTemporary resultTag;
2139     GPRReg resultTagReg = InvalidGPRReg;
2140     if (node->arrayMode().isOutOfBounds()) {
2141         GPRTemporary realResultTag(this);
2142         resultTag.adopt(realResultTag);
2143         resultTagReg = resultTag.gpr();
2144     }
2145 #endif
2146
2147     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2148
2149     // unsigned comparison so we can filter out negative indices and indices that are too large
2150     JITCompiler::Jump outOfBounds = m_jit.branch32(
2151         MacroAssembler::AboveOrEqual, propertyReg,
2152         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2153     if (node->arrayMode().isInBounds())
2154         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2155
2156     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2157
2158     // Load the character into scratchReg
2159     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2160
2161     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2162     JITCompiler::Jump cont8Bit = m_jit.jump();
2163
2164     is16Bit.link(&m_jit);
2165
2166     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2167
2168     JITCompiler::Jump bigCharacter =
2169         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2170
2171     // 8 bit string values don't need the isASCII check.
2172     cont8Bit.link(&m_jit);
2173
2174     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2175     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2176     m_jit.loadPtr(scratchReg, scratchReg);
2177
2178     addSlowPathGenerator(
2179         slowPathCall(
2180             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2181
2182     if (node->arrayMode().isOutOfBounds()) {
2183 #if USE(JSVALUE32_64)
2184         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2185 #endif
2186
2187         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2188         bool prototypeChainIsSane = false;
2189         if (globalObject->stringPrototypeChainIsSane()) {
2190             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2191             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2192             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2193             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2194             // indexed properties either.
2195             // https://bugs.webkit.org/show_bug.cgi?id=144668
2196             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure());
2197             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure());
2198             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2199         }
2200         if (prototypeChainIsSane) {
2201 #if USE(JSVALUE64)
2202             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2203                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2204 #else
2205             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2206                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2207                 baseReg, propertyReg));
2208 #endif
2209         } else {
2210 #if USE(JSVALUE64)
2211             addSlowPathGenerator(
2212                 slowPathCall(
2213                     outOfBounds, this, operationGetByValStringInt,
2214                     scratchReg, baseReg, propertyReg));
2215 #else
2216             addSlowPathGenerator(
2217                 slowPathCall(
2218                     outOfBounds, this, operationGetByValStringInt,
2219                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2220 #endif
2221         }
2222         
2223 #if USE(JSVALUE64)
2224         jsValueResult(scratchReg, m_currentNode);
2225 #else
2226         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2227 #endif
2228     } else
2229         cellResult(scratchReg, m_currentNode);
2230 }
2231
2232 void SpeculativeJIT::compileFromCharCode(Node* node)
2233 {
2234     Edge& child = node->child1();
2235     if (child.useKind() == UntypedUse) {
2236         JSValueOperand opr(this, child);
2237         JSValueRegs oprRegs = opr.jsValueRegs();
2238
2239         flushRegisters();
2240         JSValueRegsFlushedCallResult result(this);
2241         JSValueRegs resultRegs = result.regs();
2242         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2243         m_jit.exceptionCheck();
2244         
2245         jsValueResult(resultRegs, node);
2246         return;
2247     }
2248
2249     SpeculateStrictInt32Operand property(this, child);
2250     GPRReg propertyReg = property.gpr();
2251     GPRTemporary smallStrings(this);
2252     GPRTemporary scratch(this);
2253     GPRReg scratchReg = scratch.gpr();
2254     GPRReg smallStringsReg = smallStrings.gpr();
2255
2256     JITCompiler::JumpList slowCases;
2257     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2258     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2259     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2260
2261     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2262     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2263     cellResult(scratchReg, m_currentNode);
2264 }
2265
2266 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2267 {
2268     VirtualRegister virtualRegister = node->virtualRegister();
2269     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2270
2271     switch (info.registerFormat()) {
2272     case DataFormatStorage:
2273         RELEASE_ASSERT_NOT_REACHED();
2274
2275     case DataFormatBoolean:
2276     case DataFormatCell:
2277         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2278         return GeneratedOperandTypeUnknown;
2279
2280     case DataFormatNone:
2281     case DataFormatJSCell:
2282     case DataFormatJS:
2283     case DataFormatJSBoolean:
2284     case DataFormatJSDouble:
2285         return GeneratedOperandJSValue;
2286
2287     case DataFormatJSInt32:
2288     case DataFormatInt32:
2289         return GeneratedOperandInteger;
2290
2291     default:
2292         RELEASE_ASSERT_NOT_REACHED();
2293         return GeneratedOperandTypeUnknown;
2294     }
2295 }
2296
2297 void SpeculativeJIT::compileValueToInt32(Node* node)
2298 {
2299     switch (node->child1().useKind()) {
2300 #if USE(JSVALUE64)
2301     case Int52RepUse: {
2302         SpeculateStrictInt52Operand op1(this, node->child1());
2303         GPRTemporary result(this, Reuse, op1);
2304         GPRReg op1GPR = op1.gpr();
2305         GPRReg resultGPR = result.gpr();
2306         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2307         int32Result(resultGPR, node, DataFormatInt32);
2308         return;
2309     }
2310 #endif // USE(JSVALUE64)
2311         
2312     case DoubleRepUse: {
2313         GPRTemporary result(this);
2314         SpeculateDoubleOperand op1(this, node->child1());
2315         FPRReg fpr = op1.fpr();
2316         GPRReg gpr = result.gpr();
2317         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2318         
2319         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2320             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2321         
2322         int32Result(gpr, node);
2323         return;
2324     }
2325     
2326     case NumberUse:
2327     case NotCellUse: {
2328         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2329         case GeneratedOperandInteger: {
2330             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2331             GPRTemporary result(this, Reuse, op1);
2332             m_jit.move(op1.gpr(), result.gpr());
2333             int32Result(result.gpr(), node, op1.format());
2334             return;
2335         }
2336         case GeneratedOperandJSValue: {
2337             GPRTemporary result(this);
2338 #if USE(JSVALUE64)
2339             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2340
2341             GPRReg gpr = op1.gpr();
2342             GPRReg resultGpr = result.gpr();
2343             FPRTemporary tempFpr(this);
2344             FPRReg fpr = tempFpr.fpr();
2345
2346             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2347             JITCompiler::JumpList converted;
2348
2349             if (node->child1().useKind() == NumberUse) {
2350                 DFG_TYPE_CHECK(
2351                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2352                     m_jit.branchTest64(
2353                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2354             } else {
2355                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2356                 
2357                 DFG_TYPE_CHECK(
2358                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2359                 
2360                 // It's not a cell: so true turns into 1 and all else turns into 0.
2361                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2362                 converted.append(m_jit.jump());
2363                 
2364                 isNumber.link(&m_jit);
2365             }
2366
2367             // First, if we get here we have a double encoded as a JSValue
2368             unboxDouble(gpr, resultGpr, fpr);
2369
2370             silentSpillAllRegisters(resultGpr);
2371             callOperation(operationToInt32, resultGpr, fpr);
2372             silentFillAllRegisters();
2373
2374             converted.append(m_jit.jump());
2375
2376             isInteger.link(&m_jit);
2377             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2378
2379             converted.link(&m_jit);
2380 #else
2381             Node* childNode = node->child1().node();
2382             VirtualRegister virtualRegister = childNode->virtualRegister();
2383             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2384
2385             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2386
2387             GPRReg payloadGPR = op1.payloadGPR();
2388             GPRReg resultGpr = result.gpr();
2389         
2390             JITCompiler::JumpList converted;
2391
2392             if (info.registerFormat() == DataFormatJSInt32)
2393                 m_jit.move(payloadGPR, resultGpr);
2394             else {
2395                 GPRReg tagGPR = op1.tagGPR();
2396                 FPRTemporary tempFpr(this);
2397                 FPRReg fpr = tempFpr.fpr();
2398                 FPRTemporary scratch(this);
2399
2400                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2401
2402                 if (node->child1().useKind() == NumberUse) {
2403                     DFG_TYPE_CHECK(
2404                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2405                         m_jit.branch32(
2406                             MacroAssembler::AboveOrEqual, tagGPR,
2407                             TrustedImm32(JSValue::LowestTag)));
2408                 } else {
2409                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2410                     
2411                     DFG_TYPE_CHECK(
2412                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2413                         m_jit.branchIfCell(op1.jsValueRegs()));
2414                     
2415                     // It's not a cell: so true turns into 1 and all else turns into 0.
2416                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2417                     m_jit.move(TrustedImm32(0), resultGpr);
2418                     converted.append(m_jit.jump());
2419                     
2420                     isBoolean.link(&m_jit);
2421                     m_jit.move(payloadGPR, resultGpr);
2422                     converted.append(m_jit.jump());
2423                     
2424                     isNumber.link(&m_jit);
2425                 }
2426
2427                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2428
2429                 silentSpillAllRegisters(resultGpr);
2430                 callOperation(operationToInt32, resultGpr, fpr);
2431                 silentFillAllRegisters();
2432
2433                 converted.append(m_jit.jump());
2434
2435                 isInteger.link(&m_jit);
2436                 m_jit.move(payloadGPR, resultGpr);
2437
2438                 converted.link(&m_jit);
2439             }
2440 #endif
2441             int32Result(resultGpr, node);
2442             return;
2443         }
2444         case GeneratedOperandTypeUnknown:
2445             RELEASE_ASSERT(!m_compileOkay);
2446             return;
2447         }
2448         RELEASE_ASSERT_NOT_REACHED();
2449         return;
2450     }
2451     
2452     default:
2453         ASSERT(!m_compileOkay);
2454         return;
2455     }
2456 }
2457
2458 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2459 {
2460     if (doesOverflow(node->arithMode())) {
2461         if (enableInt52()) {
2462             SpeculateInt32Operand op1(this, node->child1());
2463             GPRTemporary result(this, Reuse, op1);
2464             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2465             strictInt52Result(result.gpr(), node);
2466             return;
2467         }
2468         SpeculateInt32Operand op1(this, node->child1());
2469         FPRTemporary result(this);
2470             
2471         GPRReg inputGPR = op1.gpr();
2472         FPRReg outputFPR = result.fpr();
2473             
2474         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2475             
2476         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2477         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2478         positive.link(&m_jit);
2479             
2480         doubleResult(outputFPR, node);
2481         return;
2482     }
2483     
2484     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2485
2486     SpeculateInt32Operand op1(this, node->child1());
2487     GPRTemporary result(this);
2488
2489     m_jit.move(op1.gpr(), result.gpr());
2490
2491     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2492
2493     int32Result(result.gpr(), node, op1.format());
2494 }
2495
2496 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2497 {
2498     SpeculateDoubleOperand op1(this, node->child1());
2499     FPRTemporary scratch(this);
2500     GPRTemporary result(this);
2501     
2502     FPRReg valueFPR = op1.fpr();
2503     FPRReg scratchFPR = scratch.fpr();
2504     GPRReg resultGPR = result.gpr();
2505
2506     JITCompiler::JumpList failureCases;
2507     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2508     m_jit.branchConvertDoubleToInt32(
2509         valueFPR, resultGPR, failureCases, scratchFPR,
2510         shouldCheckNegativeZero(node->arithMode()));
2511     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2512
2513     int32Result(resultGPR, node);
2514 }
2515
2516 void SpeculativeJIT::compileDoubleRep(Node* node)
2517 {
2518     switch (node->child1().useKind()) {
2519     case RealNumberUse: {
2520         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2521         FPRTemporary result(this);
2522         
2523         JSValueRegs op1Regs = op1.jsValueRegs();
2524         FPRReg resultFPR = result.fpr();
2525         
2526 #if USE(JSVALUE64)
2527         GPRTemporary temp(this);
2528         GPRReg tempGPR = temp.gpr();
2529         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2530 #else
2531         FPRTemporary temp(this);
2532         FPRReg tempFPR = temp.fpr();
2533         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2534 #endif
2535         
2536         JITCompiler::Jump done = m_jit.branchDouble(
2537             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2538         
2539         DFG_TYPE_CHECK(
2540             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2541         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2542         
2543         done.link(&m_jit);
2544         
2545         doubleResult(resultFPR, node);
2546         return;
2547     }
2548     
2549     case NotCellUse:
2550     case NumberUse: {
2551         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2552
2553         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2554         if (isInt32Speculation(possibleTypes)) {
2555             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2556             FPRTemporary result(this);
2557             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2558             doubleResult(result.fpr(), node);
2559             return;
2560         }
2561
2562         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2563         FPRTemporary result(this);
2564
2565 #if USE(JSVALUE64)
2566         GPRTemporary temp(this);
2567
2568         GPRReg op1GPR = op1.gpr();
2569         GPRReg tempGPR = temp.gpr();
2570         FPRReg resultFPR = result.fpr();
2571         JITCompiler::JumpList done;
2572
2573         JITCompiler::Jump isInteger = m_jit.branch64(
2574             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2575
2576         if (node->child1().useKind() == NotCellUse) {
2577             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2578             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2579
2580             static const double zero = 0;
2581             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2582
2583             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2584             done.append(isNull);
2585
2586             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2587                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2588
2589             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2590             static const double one = 1;
2591             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2592             done.append(m_jit.jump());
2593             done.append(isFalse);
2594
2595             isUndefined.link(&m_jit);
2596             static const double NaN = PNaN;
2597             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2598             done.append(m_jit.jump());
2599
2600             isNumber.link(&m_jit);
2601         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2602             typeCheck(
2603                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2604                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2605         }
2606
2607         unboxDouble(op1GPR, tempGPR, resultFPR);
2608         done.append(m_jit.jump());
2609     
2610         isInteger.link(&m_jit);
2611         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2612         done.link(&m_jit);
2613 #else // USE(JSVALUE64) -> this is the 32_64 case
2614         FPRTemporary temp(this);
2615     
2616         GPRReg op1TagGPR = op1.tagGPR();
2617         GPRReg op1PayloadGPR = op1.payloadGPR();
2618         FPRReg tempFPR = temp.fpr();
2619         FPRReg resultFPR = result.fpr();
2620         JITCompiler::JumpList done;
2621     
2622         JITCompiler::Jump isInteger = m_jit.branch32(
2623             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2624
2625         if (node->child1().useKind() == NotCellUse) {
2626             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2627             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2628
2629             static const double zero = 0;
2630             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2631
2632             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2633             done.append(isNull);
2634
2635             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2636
2637             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2638             static const double one = 1;
2639             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2640             done.append(m_jit.jump());
2641             done.append(isFalse);
2642
2643             isUndefined.link(&m_jit);
2644             static const double NaN = PNaN;
2645             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2646             done.append(m_jit.jump());
2647
2648             isNumber.link(&m_jit);
2649         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2650             typeCheck(
2651                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2652                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2653         }
2654
2655         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2656         done.append(m_jit.jump());
2657     
2658         isInteger.link(&m_jit);
2659         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2660         done.link(&m_jit);
2661 #endif // USE(JSVALUE64)
2662     
2663         doubleResult(resultFPR, node);
2664         return;
2665     }
2666         
2667 #if USE(JSVALUE64)
2668     case Int52RepUse: {
2669         SpeculateStrictInt52Operand value(this, node->child1());
2670         FPRTemporary result(this);
2671         
2672         GPRReg valueGPR = value.gpr();
2673         FPRReg resultFPR = result.fpr();
2674
2675         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2676         
2677         doubleResult(resultFPR, node);
2678         return;
2679     }
2680 #endif // USE(JSVALUE64)
2681         
2682     default:
2683         RELEASE_ASSERT_NOT_REACHED();
2684         return;
2685     }
2686 }
2687
2688 void SpeculativeJIT::compileValueRep(Node* node)
2689 {
2690     switch (node->child1().useKind()) {
2691     case DoubleRepUse: {
2692         SpeculateDoubleOperand value(this, node->child1());
2693         JSValueRegsTemporary result(this);
2694         
2695         FPRReg valueFPR = value.fpr();
2696         JSValueRegs resultRegs = result.regs();
2697         
2698         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2699         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2700         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2701         // local was purified.
2702         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2703             m_jit.purifyNaN(valueFPR);
2704
2705         boxDouble(valueFPR, resultRegs);
2706         
2707         jsValueResult(resultRegs, node);
2708         return;
2709     }
2710         
2711 #if USE(JSVALUE64)
2712     case Int52RepUse: {
2713         SpeculateStrictInt52Operand value(this, node->child1());
2714         GPRTemporary result(this);
2715         
2716         GPRReg valueGPR = value.gpr();
2717         GPRReg resultGPR = result.gpr();
2718         
2719         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2720         
2721         jsValueResult(resultGPR, node);
2722         return;
2723     }
2724 #endif // USE(JSVALUE64)
2725         
2726     default:
2727         RELEASE_ASSERT_NOT_REACHED();
2728         return;
2729     }
2730 }
2731
2732 static double clampDoubleToByte(double d)
2733 {
2734     d += 0.5;
2735     if (!(d > 0))
2736         d = 0;
2737     else if (d > 255)
2738         d = 255;
2739     return d;
2740 }
2741
2742 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2743 {
2744     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2745     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2746     jit.xorPtr(result, result);
2747     MacroAssembler::Jump clamped = jit.jump();
2748     tooBig.link(&jit);
2749     jit.move(JITCompiler::TrustedImm32(255), result);
2750     clamped.link(&jit);
2751     inBounds.link(&jit);
2752 }
2753
2754 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2755 {
2756     // Unordered compare so we pick up NaN
2757     static const double zero = 0;
2758     static const double byteMax = 255;
2759     static const double half = 0.5;
2760     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2761     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2762     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2763     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2764     
2765     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2766     // FIXME: This should probably just use a floating point round!
2767     // https://bugs.webkit.org/show_bug.cgi?id=72054
2768     jit.addDouble(source, scratch);
2769     jit.truncateDoubleToInt32(scratch, result);   
2770     MacroAssembler::Jump truncatedInt = jit.jump();
2771     
2772     tooSmall.link(&jit);
2773     jit.xorPtr(result, result);
2774     MacroAssembler::Jump zeroed = jit.jump();
2775     
2776     tooBig.link(&jit);
2777     jit.move(JITCompiler::TrustedImm32(255), result);
2778     
2779     truncatedInt.link(&jit);
2780     zeroed.link(&jit);
2781
2782 }
2783
2784 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2785 {
2786     if (node->op() == PutByValAlias)
2787         return JITCompiler::Jump();
2788     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2789         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2790     if (view) {
2791         uint32_t length = view->length();
2792         Node* indexNode = m_jit.graph().child(node, 1).node();
2793         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2794             return JITCompiler::Jump();
2795         return m_jit.branch32(
2796             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2797     }
2798     return m_jit.branch32(
2799         MacroAssembler::AboveOrEqual, indexGPR,
2800         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2801 }
2802
2803 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2804 {
2805     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2806     if (!jump.isSet())
2807         return;
2808     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2809 }
2810
2811 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2812 {
2813     JITCompiler::Jump done;
2814     if (outOfBounds.isSet()) {
2815         done = m_jit.jump();
2816         if (node->arrayMode().isInBounds())
2817             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2818         else {
2819             outOfBounds.link(&m_jit);
2820
2821             JITCompiler::Jump notWasteful = m_jit.branch32(
2822                 MacroAssembler::NotEqual,
2823                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2824                 TrustedImm32(WastefulTypedArray));
2825
2826             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2827                 MacroAssembler::Zero,
2828                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2829             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2830             notWasteful.link(&m_jit);
2831         }
2832     }
2833     return done;
2834 }
2835
2836 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2837 {
2838     switch (elementSize(type)) {
2839     case 1:
2840         if (isSigned(type))
2841             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2842         else
2843             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2844         break;
2845     case 2:
2846         if (isSigned(type))
2847             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2848         else
2849             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2850         break;
2851     case 4:
2852         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2853         break;
2854     default:
2855         CRASH();
2856     }
2857 }
2858
2859 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2860 {
2861     if (elementSize(type) < 4 || isSigned(type)) {
2862         int32Result(resultReg, node);
2863         return;
2864     }
2865     
2866     ASSERT(elementSize(type) == 4 && !isSigned(type));
2867     if (node->shouldSpeculateInt32() && canSpeculate) {
2868         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2869         int32Result(resultReg, node);
2870         return;
2871     }
2872     
2873 #if USE(JSVALUE64)
2874     if (node->shouldSpeculateAnyInt()) {
2875         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2876         strictInt52Result(resultReg, node);
2877         return;
2878     }
2879 #endif
2880     
2881     FPRTemporary fresult(this);
2882     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2883     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2884     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2885     positive.link(&m_jit);
2886     doubleResult(fresult.fpr(), node);
2887 }
2888
2889 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2890 {
2891     ASSERT(isInt(type));
2892     
2893     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2894     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2895     StorageOperand storage(this, m_graph.varArgChild(node, 2));
2896
2897     GPRReg baseReg = base.gpr();
2898     GPRReg propertyReg = property.gpr();
2899     GPRReg storageReg = storage.gpr();
2900
2901     GPRTemporary result(this);
2902     GPRReg resultReg = result.gpr();
2903
2904     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2905
2906     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2907     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2908     bool canSpeculate = true;
2909     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2910 }
2911
2912 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2913     GPRTemporary& value,
2914     GPRReg property,
2915 #if USE(JSVALUE32_64)
2916     GPRTemporary& propertyTag,
2917     GPRTemporary& valueTag,
2918 #endif
2919     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2920 {
2921     bool isAppropriateConstant = false;
2922     if (valueUse->isConstant()) {
2923         JSValue jsValue = valueUse->asJSValue();
2924         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2925         SpeculatedType actualType = speculationFromValue(jsValue);
2926         isAppropriateConstant = (expectedType | actualType) == expectedType;
2927     }
2928     
2929     if (isAppropriateConstant) {
2930         JSValue jsValue = valueUse->asJSValue();
2931         if (!jsValue.isNumber()) {
2932             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2933             return false;
2934         }
2935         double d = jsValue.asNumber();
2936         if (isClamped)
2937             d = clampDoubleToByte(d);
2938         GPRTemporary scratch(this);
2939         GPRReg scratchReg = scratch.gpr();
2940         m_jit.move(Imm32(toInt32(d)), scratchReg);
2941         value.adopt(scratch);
2942     } else {
2943         switch (valueUse.useKind()) {
2944         case Int32Use: {
2945             SpeculateInt32Operand valueOp(this, valueUse);
2946             GPRTemporary scratch(this);
2947             GPRReg scratchReg = scratch.gpr();
2948             m_jit.move(valueOp.gpr(), scratchReg);
2949             if (isClamped)
2950                 compileClampIntegerToByte(m_jit, scratchReg);
2951             value.adopt(scratch);
2952             break;
2953         }
2954             
2955 #if USE(JSVALUE64)
2956         case Int52RepUse: {
2957             SpeculateStrictInt52Operand valueOp(this, valueUse);
2958             GPRTemporary scratch(this);
2959             GPRReg scratchReg = scratch.gpr();
2960             m_jit.move(valueOp.gpr(), scratchReg);
2961             if (isClamped) {
2962                 MacroAssembler::Jump inBounds = m_jit.branch64(
2963                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2964                 MacroAssembler::Jump tooBig = m_jit.branch64(
2965                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2966                 m_jit.move(TrustedImm32(0), scratchReg);
2967                 MacroAssembler::Jump clamped = m_jit.jump();
2968                 tooBig.link(&m_jit);
2969                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2970                 clamped.link(&m_jit);
2971                 inBounds.link(&m_jit);
2972             }
2973             value.adopt(scratch);
2974             break;
2975         }
2976 #endif // USE(JSVALUE64)
2977             
2978         case DoubleRepUse: {
2979             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
2980             if (isClamped) {
2981                 SpeculateDoubleOperand valueOp(this, valueUse);
2982                 GPRTemporary result(this);
2983                 FPRTemporary floatScratch(this);
2984                 FPRReg fpr = valueOp.fpr();
2985                 GPRReg gpr = result.gpr();
2986                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
2987                 value.adopt(result);
2988             } else {
2989 #if USE(JSVALUE32_64)
2990                 GPRTemporary realPropertyTag(this);
2991                 propertyTag.adopt(realPropertyTag);
2992                 GPRReg propertyTagGPR = propertyTag.gpr();
2993
2994                 GPRTemporary realValueTag(this);
2995                 valueTag.adopt(realValueTag);
2996                 GPRReg valueTagGPR = valueTag.gpr();
2997 #endif
2998                 SpeculateDoubleOperand valueOp(this, valueUse);
2999                 GPRTemporary result(this);
3000                 FPRReg fpr = valueOp.fpr();
3001                 GPRReg gpr = result.gpr();
3002                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3003                 m_jit.xorPtr(gpr, gpr);
3004                 MacroAssembler::JumpList fixed(m_jit.jump());
3005                 notNaN.link(&m_jit);
3006
3007                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3008                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3009
3010 #if USE(JSVALUE64)
3011                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3012                 boxDouble(fpr, gpr);
3013 #else
3014                 UNUSED_PARAM(property);
3015                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3016                 boxDouble(fpr, valueTagGPR, gpr);
3017 #endif
3018                 slowPathCases.append(m_jit.jump());
3019
3020                 fixed.link(&m_jit);
3021                 value.adopt(result);
3022             }
3023             break;
3024         }
3025             
3026         default:
3027             RELEASE_ASSERT_NOT_REACHED();
3028             break;
3029         }
3030     }
3031     return true;
3032 }
3033
3034 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3035 {
3036     ASSERT(isInt(type));
3037     
3038     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3039     GPRReg storageReg = storage.gpr();
3040     
3041     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3042     
3043     GPRTemporary value;
3044 #if USE(JSVALUE32_64)
3045     GPRTemporary propertyTag;
3046     GPRTemporary valueTag;
3047 #endif
3048
3049     JITCompiler::JumpList slowPathCases;
3050     
3051     bool result = getIntTypedArrayStoreOperand(
3052         value, property,
3053 #if USE(JSVALUE32_64)
3054         propertyTag, valueTag,
3055 #endif
3056         valueUse, slowPathCases, isClamped(type));
3057     if (!result) {
3058         noResult(node);
3059         return;
3060     }
3061
3062     GPRReg valueGPR = value.gpr();
3063 #if USE(JSVALUE32_64)
3064     GPRReg propertyTagGPR = propertyTag.gpr();
3065     GPRReg valueTagGPR = valueTag.gpr();
3066 #endif
3067
3068     ASSERT_UNUSED(valueGPR, valueGPR != property);
3069     ASSERT(valueGPR != base);
3070     ASSERT(valueGPR != storageReg);
3071     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3072
3073     switch (elementSize(type)) {
3074     case 1:
3075         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3076         break;
3077     case 2:
3078         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3079         break;
3080     case 4:
3081         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3082         break;
3083     default:
3084         CRASH();
3085     }
3086
3087     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3088     if (done.isSet())
3089         done.link(&m_jit);
3090
3091     if (!slowPathCases.empty()) {
3092 #if USE(JSVALUE64)
3093         if (node->op() == PutByValDirect) {
3094             addSlowPathGenerator(slowPathCall(
3095                 slowPathCases, this,
3096                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3097                 NoResult, base, property, valueGPR));
3098         } else {
3099             addSlowPathGenerator(slowPathCall(
3100                 slowPathCases, this,
3101                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3102                 NoResult, base, property, valueGPR));
3103         }
3104 #else // not USE(JSVALUE64)
3105         if (node->op() == PutByValDirect) {
3106             addSlowPathGenerator(slowPathCall(
3107                 slowPathCases, this,
3108                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3109                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3110         } else {
3111             addSlowPathGenerator(slowPathCall(
3112                 slowPathCases, this,
3113                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3114                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3115         }
3116 #endif
3117     }
3118     
3119     noResult(node);
3120 }
3121
3122 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3123 {
3124     ASSERT(isFloat(type));
3125     
3126     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3127     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3128     StorageOperand storage(this, m_graph.varArgChild(node, 2));
3129
3130     GPRReg baseReg = base.gpr();
3131     GPRReg propertyReg = property.gpr();
3132     GPRReg storageReg = storage.gpr();
3133
3134     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3135
3136     FPRTemporary result(this);
3137     FPRReg resultReg = result.fpr();
3138     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3139     switch (elementSize(type)) {
3140     case 4:
3141         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3142         m_jit.convertFloatToDouble(resultReg, resultReg);
3143         break;
3144     case 8: {
3145         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3146         break;
3147     }
3148     default:
3149         RELEASE_ASSERT_NOT_REACHED();
3150     }
3151     
3152     doubleResult(resultReg, node);
3153 }
3154
3155 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3156 {
3157     ASSERT(isFloat(type));
3158     
3159     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3160     GPRReg storageReg = storage.gpr();
3161     
3162     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3163     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3164
3165     SpeculateDoubleOperand valueOp(this, valueUse);
3166     FPRTemporary scratch(this);
3167     FPRReg valueFPR = valueOp.fpr();
3168     FPRReg scratchFPR = scratch.fpr();
3169
3170     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3171     
3172     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3173     
3174     switch (elementSize(type)) {
3175     case 4: {
3176         m_jit.moveDouble(valueFPR, scratchFPR);
3177         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3178         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3179         break;
3180     }
3181     case 8:
3182         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3183         break;
3184     default:
3185         RELEASE_ASSERT_NOT_REACHED();
3186     }
3187
3188     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3189     if (done.isSet())
3190         done.link(&m_jit);
3191     noResult(node);
3192 }
3193
3194 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3195 {
3196     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3197     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3198
3199     GPRReg arg1GPR = arg1.gpr();
3200     GPRReg arg2GPR = arg2.gpr();
3201
3202     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3203     speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3204
3205     flushRegisters();
3206     JSValueRegsFlushedCallResult result(this);
3207     JSValueRegs resultRegs = result.regs();
3208     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3209     m_jit.exceptionCheck();
3210
3211     jsValueResult(resultRegs, node);
3212 }
3213
3214 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3215 {
3216     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3217     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3218
3219     GPRReg arg1GPR = arg1.gpr();
3220     GPRReg arg2GPR = arg2.gpr();
3221
3222     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3223     speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3224
3225     flushRegisters();
3226     JSValueRegsFlushedCallResult result(this);
3227     JSValueRegs resultRegs = result.regs();
3228     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3229     m_jit.exceptionCheck();
3230
3231     jsValueResult(resultRegs, node);
3232 }
3233
3234 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3235 {
3236     SpeculateCellOperand arg1(this, child1);
3237     SpeculateCellOperand arg2(this, child2);
3238     JSValueOperand arg3(this, child3);
3239
3240     GPRReg arg1GPR = arg1.gpr();
3241     GPRReg arg2GPR = arg2.gpr();
3242     JSValueRegs arg3Regs = arg3.jsValueRegs();
3243
3244     speculateString(child2, arg2GPR);
3245
3246     flushRegisters();
3247     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3248     m_jit.exceptionCheck();
3249
3250     noResult(node);
3251 }
3252
3253 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3254 {
3255     SpeculateCellOperand arg1(this, child1);
3256     SpeculateCellOperand arg2(this, child2);
3257     JSValueOperand arg3(this, child3);
3258
3259     GPRReg arg1GPR = arg1.gpr();
3260     GPRReg arg2GPR = arg2.gpr();
3261     JSValueRegs arg3Regs = arg3.jsValueRegs();
3262
3263     speculateSymbol(child2, arg2GPR);
3264
3265     flushRegisters();
3266     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3267     m_jit.exceptionCheck();
3268
3269     noResult(node);
3270 }
3271
3272 void SpeculativeJIT::compileGetByValWithThis(Node* node)
3273 {
3274     JSValueOperand base(this, node->child1());
3275     JSValueRegs baseRegs = base.jsValueRegs();
3276     JSValueOperand thisValue(this, node->child2());
3277     JSValueRegs thisValueRegs = thisValue.jsValueRegs();
3278     JSValueOperand subscript(this, node->child3());
3279     JSValueRegs subscriptRegs = subscript.jsValueRegs();
3280
3281     flushRegisters();
3282     JSValueRegsFlushedCallResult result(this);
3283     JSValueRegs resultRegs = result.regs();
3284     callOperation(operationGetByValWithThis, resultRegs, baseRegs, thisValueRegs, subscriptRegs);
3285     m_jit.exceptionCheck();
3286
3287     jsValueResult(resultRegs, node);
3288 }
3289
3290 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg, GPRReg scratch3Reg)
3291 {
3292     // Check that prototype is an object.
3293     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
3294     
3295     // Initialize scratchReg with the value being checked.
3296     m_jit.move(valueReg, scratchReg);
3297     
3298     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
3299     MacroAssembler::Label loop(&m_jit);
3300     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
3301         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
3302     m_jit.emitLoadStructure(*m_jit.vm(), scratchReg, scratch3Reg, scratch2Reg);
3303 #if USE(JSVALUE64)
3304     m_jit.load64(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset()), scratch3Reg);
3305     auto hasMonoProto = m_jit.branchTest64(JITCompiler::NonZero, scratch3Reg);
3306     m_jit.load64(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset)), scratch3Reg);
3307     hasMonoProto.link(&m_jit);
3308     m_jit.move(scratch3Reg, scratchReg);
3309 #else
3310     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + TagOffset), scratch2Reg);
3311     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + PayloadOffset), scratch3Reg);
3312     auto hasMonoProto = m_jit.branch32(CCallHelpers::NotEqual, scratch2Reg, TrustedImm32(JSValue::EmptyValueTag));
3313     m_jit.load32(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), scratch3Reg);
3314     hasMonoProto.link(&m_jit);
3315     m_jit.move(scratch3Reg, scratchReg);
3316 #endif
3317
3318     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
3319 #if USE(JSVALUE64)
3320     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
3321 #else
3322     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
3323 #endif
3324     
3325     // No match - result is false.
3326 #if USE(JSVALUE64)
3327     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
3328 #else
3329     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
3330 #endif
3331     MacroAssembler::JumpList doneJumps; 
3332     doneJumps.append(m_jit.jump());
3333
3334     performDefaultHasInstance.link(&m_jit);
3335     silentSpillAllRegisters(scratchReg);
3336     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg); 
3337     silentFillAllRegisters();
3338     m_jit.exceptionCheck();
3339 #if USE(JSVALUE64)
3340     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
3341 #endif
3342     doneJumps.append(m_jit.jump());
3343     
3344     isInstance.link(&m_jit);
3345 #if USE(JSVALUE64)
3346     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), scratchReg);
3347 #else
3348     m_jit.move(MacroAssembler::TrustedImm32(1), scratchReg);
3349 #endif
3350     
3351     doneJumps.link(&m_jit);
3352 }
3353
3354 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3355 {
3356     SpeculateCellOperand base(this, node->child1());
3357
3358     GPRReg baseGPR = base.gpr();
3359
3360     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3361
3362     noResult(node);
3363 }
3364
3365 void SpeculativeJIT::compileParseInt(Node* node)
3366 {
3367     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3368     if (node->child2()) {
3369         SpeculateInt32Operand radix(this, node->child2());
3370         GPRReg radixGPR = radix.gpr();
3371         if (node->child1().useKind() == UntypedUse) {
3372             JSValueOperand value(this, node->child1());
3373             JSValueRegs valueRegs = value.jsValueRegs();
3374
3375             flushRegisters();
3376             JSValueRegsFlushedCallResult result(this);
3377             JSValueRegs resultRegs = result.regs();
3378             callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3379             m_jit.exceptionCheck();
3380             jsValueResult(resultRegs, node);
3381             return;
3382         }
3383
3384         SpeculateCellOperand value(this, node->child1());
3385         GPRReg valueGPR = value.gpr();
3386         speculateString(node->child1(), valueGPR);
3387
3388         flushRegisters();
3389         JSValueRegsFlushedCallResult result(this);
3390         JSValueRegs resultRegs = result.regs();
3391         callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3392         m_jit.exceptionCheck();
3393         jsValueResult(resultRegs, node);
3394         return;
3395     }
3396
3397     if (node->child1().useKind() == UntypedUse) {
3398         JSValueOperand value(this, node->child1());
3399         JSValueRegs valueRegs = value.jsValueRegs();
3400
3401         flushRegisters();
3402         JSValueRegsFlushedCallResult result(this);
3403         JSValueRegs resultRegs = result.regs();
3404         callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3405         m_jit.exceptionCheck();
3406         jsValueResult(resultRegs, node);
3407         return;
3408     }
3409
3410     SpeculateCellOperand value(this, node->child1());
3411     GPRReg valueGPR = value.gpr();
3412     speculateString(node->child1(), valueGPR);
3413
3414     flushRegisters();
3415     JSValueRegsFlushedCallResult result(this);
3416     JSValueRegs resultRegs = result.regs();
3417     callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3418     m_jit.exceptionCheck();
3419     jsValueResult(resultRegs, node);
3420 }
3421
3422 void SpeculativeJIT::compileInstanceOf(Node* node)
3423 {
3424     if (node->child1().useKind() == UntypedUse) {
3425         // It might not be a cell. Speculate less aggressively.
3426         // Or: it might only be used once (i.e. by us), so we get zero benefit
3427         // from speculating any more aggressively than we absolutely need to.
3428         
3429         JSValueOperand value(this, node->child1());
3430         SpeculateCellOperand prototype(this, node->child2());
3431         GPRTemporary scratch(this);
3432         GPRTemporary scratch2(this);
3433         GPRTemporary scratch3(this);
3434         
3435         GPRReg prototypeReg = prototype.gpr();
3436         GPRReg scratchReg = scratch.gpr();
3437         GPRReg scratch2Reg = scratch2.gpr();
3438         GPRReg scratch3Reg = scratch3.gpr();
3439         
3440         MacroAssembler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs());
3441         GPRReg valueReg = value.jsValueRegs().payloadGPR();
3442         moveFalseTo(scratchReg);
3443
3444         MacroAssembler::Jump done = m_jit.jump();
3445         
3446         isCell.link(&m_jit);
3447         
3448         compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3449         
3450         done.link(&m_jit);
3451
3452         blessedBooleanResult(scratchReg, node);
3453         return;
3454     }
3455     
3456     SpeculateCellOperand value(this, node->child1());
3457     SpeculateCellOperand prototype(this, node->child2());
3458     
3459     GPRTemporary scratch(this);
3460     GPRTemporary scratch2(this);
3461     GPRTemporary scratch3(this);
3462     
3463     GPRReg valueReg = value.gpr();
3464     GPRReg prototypeReg = prototype.gpr();
3465     GPRReg scratchReg = scratch.gpr();
3466     GPRReg scratch2Reg = scratch2.gpr();
3467     GPRReg scratch3Reg = scratch3.gpr();
3468     
3469     compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3470
3471     blessedBooleanResult(scratchReg, node);
3472 }
3473
3474 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3475 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3476 {
3477     Edge& leftChild = node->child1();
3478     Edge& rightChild = node->child2();
3479
3480     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3481         JSValueOperand left(this, leftChild);
3482         JSValueOperand right(this, rightChild);
3483         JSValueRegs leftRegs = left.jsValueRegs();
3484         JSValueRegs rightRegs = right.jsValueRegs();
3485
3486         flushRegisters();
3487         JSValueRegsFlushedCallResult result(this);
3488         JSValueRegs resultRegs = result.regs();
3489         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3490         m_jit.exceptionCheck();
3491
3492         jsValueResult(resultRegs, node);
3493         return;
3494     }
3495
3496     std::optional<JSValueOperand> left;
3497     std::optional<JSValueOperand> right;
3498
3499     JSValueRegs leftRegs;
3500     JSValueRegs rightRegs;
3501
3502 #if USE(JSVALUE64)
3503     GPRTemporary result(this);
3504     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3505     GPRTemporary scratch(this);
3506     GPRReg scratchGPR = scratch.gpr();
3507 #else
3508     GPRTemporary resultTag(this);
3509     GPRTemporary resultPayload(this);
3510     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3511     GPRReg scratchGPR = resultTag.gpr();
3512 #endif
3513
3514     SnippetOperand leftOperand;
3515     SnippetOperand rightOperand;
3516
3517     // The snippet generator does not support both operands being constant. If the left
3518     // operand is already const, we'll ignore the right operand's constness.
3519     if (leftChild->isInt32Constant())
3520         leftOperand.setConstInt32(leftChild->asInt32());
3521     else if (rightChild->isInt32Constant())
3522         rightOperand.setConstInt32(rightChild->asInt32());
3523
3524     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3525
3526     if (!leftOperand.isConst()) {
3527         left.emplace(this, leftChild);
3528         leftRegs = left->jsValueRegs();
3529     }
3530     if (!rightOperand.isConst()) {
3531         right.emplace(this, rightChild);
3532         rightRegs = right->jsValueRegs();
3533     }
3534
3535     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3536     gen.generateFastPath(m_jit);
3537
3538     ASSERT(gen.didEmitFastPath());
3539     gen.endJumpList().append(m_jit.jump());
3540
3541     gen.slowPathJumpList().link(&m_jit);
3542     silentSpillAllRegisters(resultRegs);
3543
3544     if (leftOperand.isConst()) {
3545         leftRegs = resultRegs;
3546         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3547     } else if (rightOperand.isConst()) {
3548         rightRegs = resultRegs;
3549         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3550     }
3551
3552     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3553
3554     silentFillAllRegisters();
3555     m_jit.exceptionCheck();
3556
3557     gen.endJumpList().link(&m_jit);
3558     jsValueResult(resultRegs, node);
3559 }
3560
3561 void SpeculativeJIT::compileBitwiseOp(Node* node)
3562 {
3563     NodeType op = node->op();
3564     Edge& leftChild = node->child1();
3565     Edge& rightChild = node->child2();
3566
3567     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3568         switch (op) {
3569         case BitAnd:
3570             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3571             return;
3572         case BitOr:
3573             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3574             return;
3575         case BitXor:
3576             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3577             return;
3578         default:
3579             RELEASE_ASSERT_NOT_REACHED();
3580         }
3581     }
3582
3583     if (leftChild->isInt32Constant()) {
3584         SpeculateInt32Operand op2(this, rightChild);
3585         GPRTemporary result(this, Reuse, op2);
3586
3587         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3588
3589         int32Result(result.gpr(), node);
3590
3591     } else if (rightChild->isInt32Constant()) {
3592         SpeculateInt32Operand op1(this, leftChild);
3593         GPRTemporary result(this, Reuse, op1);
3594
3595         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3596
3597         int32Result(result.gpr(), node);
3598
3599     } else {
3600         SpeculateInt32Operand op1(this, leftChild);
3601         SpeculateInt32Operand op2(this, rightChild);
3602         GPRTemporary result(this, Reuse, op1, op2);
3603         
3604         GPRReg reg1 = op1.gpr();
3605         GPRReg reg2 = op2.gpr();
3606         bitOp(op, reg1, reg2, result.gpr());
3607         
3608         int32Result(result.gpr(), node);
3609     }
3610 }
3611
3612 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3613 {
3614     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3615         ? operationValueBitRShift : operationValueBitURShift;
3616     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3617         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3618
3619     Edge& leftChild = node->child1();
3620     Edge& rightChild = node->child2();
3621
3622     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3623         JSValueOperand left(this, leftChild);
3624         JSValueOperand right(this, rightChild);
3625         JSValueRegs leftRegs = left.jsValueRegs();
3626         JSValueRegs rightRegs = right.jsValueRegs();
3627
3628         flushRegisters();
3629         JSValueRegsFlushedCallResult result(this);
3630         JSValueRegs resultRegs = result.regs();
3631         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3632         m_jit.exceptionCheck();
3633
3634         jsValueResult(resultRegs, node);
3635         return;
3636     }
3637
3638     std::optional<JSValueOperand> left;
3639     std::optional<JSValueOperand> right;
3640
3641     JSValueRegs leftRegs;
3642     JSValueRegs rightRegs;
3643
3644     FPRTemporary leftNumber(this);
3645     FPRReg leftFPR = leftNumber.fpr();
3646
3647 #if USE(JSVALUE64)
3648     GPRTemporary result(this);
3649     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3650     GPRTemporary scratch(this);
3651     GPRReg scratchGPR = scratch.gpr();
3652     FPRReg scratchFPR = InvalidFPRReg;
3653 #else
3654     GPRTemporary resultTag(this);
3655     GPRTemporary resultPayload(this);
3656     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3657     GPRReg scratchGPR = resultTag.gpr();
3658     FPRTemporary fprScratch(this);
3659     FPRReg scratchFPR = fprScratch.fpr();
3660 #endif
3661
3662     SnippetOperand leftOperand;
3663     SnippetOperand rightOperand;
3664
3665     // The snippet generator does not support both operands being constant. If the left
3666     // operand is already const, we'll ignore the right operand's constness.
3667     if (leftChild->isInt32Constant())
3668         leftOperand.setConstInt32(leftChild->asInt32());
3669     else if (rightChild->isInt32Constant())
3670         rightOperand.setConstInt32(rightChild->asInt32());
3671
3672     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3673
3674     if (!leftOperand.isConst()) {
3675         left.emplace(this, leftChild);
3676         leftRegs = left->jsValueRegs();
3677     }
3678     if (!rightOperand.isConst()) {
3679         right.emplace(this, rightChild);
3680         rightRegs = right->jsValueRegs();
3681     }
3682
3683     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3684         leftFPR, scratchGPR, scratchFPR, shiftType);
3685     gen.generateFastPath(m_jit);
3686
3687     ASSERT(gen.didEmitFastPath());
3688     gen.endJumpList().append(m_jit.jump());
3689
3690     gen.slowPathJumpList().link(&m_jit);
3691     silentSpillAllRegisters(resultRegs);
3692
3693     if (leftOperand.isConst()) {
3694         leftRegs = resultRegs;
3695         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3696     } else if (rightOperand.isConst()) {
3697         rightRegs = resultRegs;
3698         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3699     }
3700
3701     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3702
3703     silentFillAllRegisters();
3704     m_jit.exceptionCheck();
3705
3706     gen.endJumpList().link(&m_jit);
3707     jsValueResult(resultRegs, node);
3708     return;
3709 }
3710
3711 void SpeculativeJIT::compileShiftOp(Node* node)
3712 {
3713     NodeType op = node->op();
3714     Edge& leftChild = node->child1();
3715     Edge& rightChild = node->child2();
3716
3717     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3718         switch (op) {
3719         case BitLShift:
3720             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3721             return;
3722         case BitRShift:
3723         case BitURShift:
3724             emitUntypedRightShiftBitOp(node);
3725             return;
3726         default:
3727             RELEASE_ASSERT_NOT_REACHED();
3728         }
3729     }
3730
3731     if (rightChild->isInt32Constant()) {
3732         SpeculateInt32Operand op1(this, leftChild);
3733         GPRTemporary result(this, Reuse, op1);
3734
3735         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3736
3737         int32Result(result.gpr(), node);
3738     } else {
3739         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3740         SpeculateInt32Operand op1(this, leftChild);
3741         SpeculateInt32Operand op2(this, rightChild);
3742         GPRTemporary result(this, Reuse, op1);
3743
3744         GPRReg reg1 = op1.gpr();
3745         GPRReg reg2 = op2.gpr();
3746         shiftOp(op, reg1, reg2, result.gpr());
3747
3748         int32Result(result.gpr(), node);
3749     }
3750 }
3751
3752 void SpeculativeJIT::compileValueAdd(Node* node)
3753 {
3754     Edge& leftChild = node->child1();
3755     Edge& rightChild = node->child2();
3756
3757     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3758         JSValueOperand left(this, leftChild);
3759         JSValueOperand right(this, rightChild);
3760         JSValueRegs leftRegs = left.jsValueRegs();
3761         JSValueRegs rightRegs = right.jsValueRegs();
3762
3763         flushRegisters();
3764         JSValueRegsFlushedCallResult result(this);
3765         JSValueRegs resultRegs = result.regs();
3766         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3767         m_jit.exceptionCheck();
3768     
3769         jsValueResult(resultRegs, node);
3770         return;
3771     }
3772
3773 #if USE(JSVALUE64)
3774     bool needsScratchGPRReg = true;
3775     bool needsScratchFPRReg = false;
3776 #else
3777     bool needsScratchGPRReg = true;
3778     bool needsScratchFPRReg = true;
3779 #endif
3780
3781     CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3782     ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(node->origin.semantic.bytecodeIndex);
3783     Instruction* instruction = &baselineCodeBlock->instructions()[node->origin.semantic.bytecodeIndex];
3784     JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile, instruction);
3785     auto repatchingFunction = operationValueAddOptimize;
3786     auto nonRepatchingFunction = operationValueAdd;
3787     
3788     compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3789 }
3790
3791 template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3792 void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3793 {
3794     Edge& leftChild = node->child1();
3795     Edge& rightChild = node->child2();
3796
3797     std::optional<JSValueOperand> left;
3798     std::optional<JSValueOperand> right;
3799
3800     JSValueRegs leftRegs;
3801     JSValueRegs rightRegs;
3802
3803     FPRTemporary leftNumber(this);
3804     FPRTemporary rightNumber(this);
3805     FPRReg leftFPR = leftNumber.fpr();
3806     FPRReg rightFPR = rightNumber.fpr();
3807
3808     GPRReg scratchGPR = InvalidGPRReg;
3809     FPRReg scratchFPR = InvalidFPRReg;
3810
3811     std::optional<FPRTemporary> fprScratch;
3812     if (needsScratchFPRReg) {
3813         fprScratch.emplace(this);
3814         scratchFPR = fprScratch->fpr();
3815     }
3816
3817 #if USE(JSVALUE64)
3818     std::optional<GPRTemporary> gprScratch;
3819     if (needsScratchGPRReg) {
3820         gprScratch.emplace(this);
3821         scratchGPR = gprScratch->gpr();
3822     }
3823     GPRTemporary result(this);
3824     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3825 #else
3826     GPRTemporary resultTag(this);
3827     GPRTemporary resultPayload(this);
3828     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3829     if (needsScratchGPRReg)
3830         scratchGPR = resultRegs.tagGPR();
3831 #endif
3832
3833     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3834     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3835
3836     // The snippet generator does not support both operands being constant. If the left
3837     // operand is already const, we'll ignore the right operand's constness.
3838     if (leftChild->isInt32Constant())
3839         leftOperand.setConstInt32(leftChild->asInt32());
3840     else if (rightChild->isInt32Constant())
3841         rightOperand.setConstInt32(rightChild->asInt32());
3842
3843     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3844     ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
3845
3846     if (!Generator::isLeftOperandValidConstant(leftOperand)) {
3847         left.emplace(this, leftChild);
3848         leftRegs = left->jsValueRegs();
3849     }
3850     if (!Generator::isRightOperandValidConstant(rightOperand)) {
3851         right.emplace(this, rightChild);
3852         rightRegs = right->jsValueRegs();
3853     }
3854
3855 #if ENABLE(MATH_IC_STATS)
3856     auto inlineStart = m_jit.label();
3857 #endif
3858
3859     Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
3860     mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
3861
3862     bool shouldEmitProfiling = false;
3863     bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
3864
3865     if (generatedInline) {
3866         ASSERT(!addICGenerationState->slowPathJumps.empty());
3867
3868         Vector<SilentRegisterSavePlan> savePlans;
3869         silentSpillAllRegistersImpl(false, savePlans, resultRegs);
3870
3871         auto done = m_jit.label();
3872
3873         addSlowPathGenerator([=, savePlans = WTFMove(savePlans)] () {
3874             addICGenerationState->slowPathJumps.link(&m_jit);
3875             addICGenerationState->slowPathStart = m_jit.label();
3876 #if ENABLE(MATH_IC_STATS)
3877             auto slowPathStart = m_jit.label();
3878 #endif
3879
3880             silentSpill(savePlans);
3881
3882             auto innerLeftRegs = leftRegs;
3883             auto innerRightRegs = rightRegs;
3884             if (Generator::isLeftOperandValidConstant(leftOperand)) {
3885                 innerLeftRegs = resultRegs;
3886                 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
3887             } else if (Generator::isRightOperandValidConstant(rightOperand)) {
3888                 innerRightRegs = resultRegs;
3889                 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
3890             }
3891
3892             if (addICGenerationState->shouldSlowPathRepatch)
3893                 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
3894             else
3895                 addICGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, innerLeftRegs, innerRightRegs);
3896