b04bde2f0712b5badf15208e4278e74ad30c9600
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCapabilities.h"
37 #include "DFGMayExit.h"
38 #include "DFGOSRExitFuzz.h"
39 #include "DFGSaneStringGetByValSlowPathGenerator.h"
40 #include "DFGSlowPathGenerator.h"
41 #include "DFGSnippetParams.h"
42 #include "DirectArguments.h"
43 #include "JITAddGenerator.h"
44 #include "JITBitAndGenerator.h"
45 #include "JITBitOrGenerator.h"
46 #include "JITBitXorGenerator.h"
47 #include "JITDivGenerator.h"
48 #include "JITLeftShiftGenerator.h"
49 #include "JITMulGenerator.h"
50 #include "JITRightShiftGenerator.h"
51 #include "JITSubGenerator.h"
52 #include "JSAsyncFunction.h"
53 #include "JSAsyncGeneratorFunction.h"
54 #include "JSCInlines.h"
55 #include "JSFixedArray.h"
56 #include "JSGeneratorFunction.h"
57 #include "JSImmutableButterfly.h"
58 #include "JSLexicalEnvironment.h"
59 #include "JSPropertyNameEnumerator.h"
60 #include "LinkBuffer.h"
61 #include "RegExpConstructor.h"
62 #include "ScopedArguments.h"
63 #include "ScratchRegisterAllocator.h"
64 #include "SuperSampler.h"
65 #include "TypeProfilerLog.h"
66 #include "WeakMapImpl.h"
67 #include <wtf/BitVector.h>
68 #include <wtf/Box.h>
69 #include <wtf/MathExtras.h>
70
71 namespace JSC { namespace DFG {
72
73 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
74     : m_compileOkay(true)
75     , m_jit(jit)
76     , m_graph(m_jit.graph())
77     , m_currentNode(0)
78     , m_lastGeneratedNode(LastNodeType)
79     , m_indexInBlock(0)
80     , m_generationInfo(m_jit.graph().frameRegisterCount())
81     , m_state(m_jit.graph())
82     , m_interpreter(m_jit.graph(), m_state)
83     , m_stream(&jit.jitCode()->variableEventStream)
84     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
85 {
86 }
87
88 SpeculativeJIT::~SpeculativeJIT()
89 {
90 }
91
92 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
93 {
94     ASSERT(!isCopyOnWrite(structure->indexingMode()));
95     IndexingType indexingType = structure->indexingType();
96     bool hasIndexingHeader = hasIndexedProperties(indexingType);
97
98     unsigned inlineCapacity = structure->inlineCapacity();
99     unsigned outOfLineCapacity = structure->outOfLineCapacity();
100     
101     GPRTemporary scratch(this);
102     GPRTemporary scratch2(this);
103     GPRReg scratchGPR = scratch.gpr();
104     GPRReg scratch2GPR = scratch2.gpr();
105
106     ASSERT(vectorLength >= numElements);
107     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
108     
109     JITCompiler::JumpList slowCases;
110
111     size_t size = 0;
112     if (hasIndexingHeader)
113         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
114     size += outOfLineCapacity * sizeof(JSValue);
115
116     m_jit.move(TrustedImmPtr(nullptr), storageGPR);
117
118     if (size) {
119         if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
120             m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
121             
122             m_jit.addPtr(
123                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
124                 storageGPR);
125             
126             if (hasIndexingHeader)
127                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
128         } else
129             slowCases.append(m_jit.jump());
130     }
131
132     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
133     Allocator allocator = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
134     if (allocator) {
135         emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
136         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
137     } else
138         slowCases.append(m_jit.jump());
139
140     // I want a slow path that also loads out the storage pointer, and that's
141     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
142     // of work for a very small piece of functionality. :-/
143     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
144         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
145         structure, vectorLength));
146
147     if (numElements < vectorLength) {
148 #if USE(JSVALUE64)
149         if (hasDouble(structure->indexingType()))
150             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
151         else
152             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
153         for (unsigned i = numElements; i < vectorLength; ++i)
154             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
155 #else
156         EncodedValueDescriptor value;
157         if (hasDouble(structure->indexingType()))
158             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
159         else
160             value.asInt64 = JSValue::encode(JSValue());
161         for (unsigned i = numElements; i < vectorLength; ++i) {
162             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
163             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
164         }
165 #endif
166     }
167     
168     if (hasIndexingHeader)
169         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
170     
171     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
172     
173     m_jit.mutatorFence(*m_jit.vm());
174 }
175
176 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
177 {
178     if (inlineCallFrame && !inlineCallFrame->isVarargs())
179         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
180     else {
181         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
182         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
183         if (!includeThis)
184             m_jit.sub32(TrustedImm32(1), lengthGPR);
185     }
186 }
187
188 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
189 {
190     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
191 }
192
193 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
194 {
195     if (origin.inlineCallFrame) {
196         if (origin.inlineCallFrame->isClosureCall) {
197             m_jit.loadPtr(
198                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
199                 calleeGPR);
200         } else {
201             m_jit.move(
202                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
203                 calleeGPR);
204         }
205     } else
206         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
207 }
208
209 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
210 {
211     m_jit.addPtr(
212         TrustedImm32(
213             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
214         GPRInfo::callFrameRegister, startGPR);
215 }
216
217 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
218 {
219     if (!Options::useOSRExitFuzz()
220         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
221         || !doOSRExitFuzzing())
222         return MacroAssembler::Jump();
223     
224     MacroAssembler::Jump result;
225     
226     m_jit.pushToSave(GPRInfo::regT0);
227     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
228     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
229     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
230     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
231     unsigned at = Options::fireOSRExitFuzzAt();
232     if (at || atOrAfter) {
233         unsigned threshold;
234         MacroAssembler::RelationalCondition condition;
235         if (atOrAfter) {
236             threshold = atOrAfter;
237             condition = MacroAssembler::Below;
238         } else {
239             threshold = at;
240             condition = MacroAssembler::NotEqual;
241         }
242         MacroAssembler::Jump ok = m_jit.branch32(
243             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
244         m_jit.popToRestore(GPRInfo::regT0);
245         result = m_jit.jump();
246         ok.link(&m_jit);
247     }
248     m_jit.popToRestore(GPRInfo::regT0);
249     
250     return result;
251 }
252
253 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
254 {
255     if (!m_compileOkay)
256         return;
257     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
258     if (fuzzJump.isSet()) {
259         JITCompiler::JumpList jumpsToFail;
260         jumpsToFail.append(fuzzJump);
261         jumpsToFail.append(jumpToFail);
262         m_jit.appendExitInfo(jumpsToFail);
263     } else
264         m_jit.appendExitInfo(jumpToFail);
265     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
266 }
267
268 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
269 {
270     if (!m_compileOkay)
271         return;
272     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
273     if (fuzzJump.isSet()) {
274         JITCompiler::JumpList myJumpsToFail;
275         myJumpsToFail.append(jumpsToFail);
276         myJumpsToFail.append(fuzzJump);
277         m_jit.appendExitInfo(myJumpsToFail);
278     } else
279         m_jit.appendExitInfo(jumpsToFail);
280     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
281 }
282
283 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
284 {
285     if (!m_compileOkay)
286         return OSRExitJumpPlaceholder();
287     unsigned index = m_jit.jitCode()->osrExit.size();
288     m_jit.appendExitInfo();
289     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
290     return OSRExitJumpPlaceholder(index);
291 }
292
293 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
294 {
295     return speculationCheck(kind, jsValueSource, nodeUse.node());
296 }
297
298 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
299 {
300     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
301 }
302
303 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
304 {
305     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
306 }
307
308 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
309 {
310     if (!m_compileOkay)
311         return;
312     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
313     m_jit.appendExitInfo(jumpToFail);
314     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
315 }
316
317 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
318 {
319     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
320 }
321
322 void SpeculativeJIT::emitInvalidationPoint(Node* node)
323 {
324     if (!m_compileOkay)
325         return;
326     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
327     m_jit.jitCode()->appendOSRExit(OSRExit(
328         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
329         this, m_stream->size()));
330     info.m_replacementSource = m_jit.watchpointLabel();
331     ASSERT(info.m_replacementSource.isSet());
332     noResult(node);
333 }
334
335 void SpeculativeJIT::unreachable(Node* node)
336 {
337     m_compileOkay = false;
338     m_jit.abortWithReason(DFGUnreachableNode, node->op());
339 }
340
341 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
342 {
343     if (!m_compileOkay)
344         return;
345     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
346     m_compileOkay = false;
347     if (verboseCompilationEnabled())
348         dataLog("Bailing compilation.\n");
349 }
350
351 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
352 {
353     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
354 }
355
356 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
357 {
358     ASSERT(needsTypeCheck(edge, typesPassedThrough));
359     m_interpreter.filter(edge, typesPassedThrough);
360     speculationCheck(exitKind, source, edge.node(), jumpToFail);
361 }
362
363 RegisterSet SpeculativeJIT::usedRegisters()
364 {
365     RegisterSet result;
366     
367     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
368         GPRReg gpr = GPRInfo::toRegister(i);
369         if (m_gprs.isInUse(gpr))
370             result.set(gpr);
371     }
372     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
373         FPRReg fpr = FPRInfo::toRegister(i);
374         if (m_fprs.isInUse(fpr))
375             result.set(fpr);
376     }
377     
378     // FIXME: This is overly conservative. We could subtract out those callee-saves that we
379     // actually saved.
380     // https://bugs.webkit.org/show_bug.cgi?id=185686
381     result.merge(RegisterSet::stubUnavailableRegisters());
382     
383     return result;
384 }
385
386 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
387 {
388     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
389 }
390
391 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
392 {
393     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
394 }
395
396 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
397 {
398     for (auto& slowPathGenerator : m_slowPathGenerators) {
399         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
400         slowPathGenerator->generate(this);
401     }
402     for (auto& slowPathLambda : m_slowPathLambdas) {
403         Node* currentNode = slowPathLambda.currentNode;
404         m_currentNode = currentNode;
405         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
406         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
407         slowPathLambda.generator();
408         m_outOfLineStreamIndex = std::nullopt;
409     }
410 }
411
412 void SpeculativeJIT::clearGenerationInfo()
413 {
414     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
415         m_generationInfo[i] = GenerationInfo();
416     m_gprs = RegisterBank<GPRInfo>();
417     m_fprs = RegisterBank<FPRInfo>();
418 }
419
420 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
421 {
422     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
423     Node* node = info.node();
424     DataFormat registerFormat = info.registerFormat();
425     ASSERT(registerFormat != DataFormatNone);
426     ASSERT(registerFormat != DataFormatDouble);
427         
428     SilentSpillAction spillAction;
429     SilentFillAction fillAction;
430         
431     if (!info.needsSpill())
432         spillAction = DoNothingForSpill;
433     else {
434 #if USE(JSVALUE64)
435         ASSERT(info.gpr() == source);
436         if (registerFormat == DataFormatInt32)
437             spillAction = Store32Payload;
438         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
439             spillAction = StorePtr;
440         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
441             spillAction = Store64;
442         else {
443             ASSERT(registerFormat & DataFormatJS);
444             spillAction = Store64;
445         }
446 #elif USE(JSVALUE32_64)
447         if (registerFormat & DataFormatJS) {
448             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
449             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
450         } else {
451             ASSERT(info.gpr() == source);
452             spillAction = Store32Payload;
453         }
454 #endif
455     }
456         
457     if (registerFormat == DataFormatInt32) {
458         ASSERT(info.gpr() == source);
459         ASSERT(isJSInt32(info.registerFormat()));
460         if (node->hasConstant()) {
461             ASSERT(node->isInt32Constant());
462             fillAction = SetInt32Constant;
463         } else
464             fillAction = Load32Payload;
465     } else if (registerFormat == DataFormatBoolean) {
466 #if USE(JSVALUE64)
467         RELEASE_ASSERT_NOT_REACHED();
468 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
469         fillAction = DoNothingForFill;
470 #endif
471 #elif USE(JSVALUE32_64)
472         ASSERT(info.gpr() == source);
473         if (node->hasConstant()) {
474             ASSERT(node->isBooleanConstant());
475             fillAction = SetBooleanConstant;
476         } else
477             fillAction = Load32Payload;
478 #endif
479     } else if (registerFormat == DataFormatCell) {
480         ASSERT(info.gpr() == source);
481         if (node->hasConstant()) {
482             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
483             node->asCell(); // To get the assertion.
484             fillAction = SetCellConstant;
485         } else {
486 #if USE(JSVALUE64)
487             fillAction = LoadPtr;
488 #else
489             fillAction = Load32Payload;
490 #endif
491         }
492     } else if (registerFormat == DataFormatStorage) {
493         ASSERT(info.gpr() == source);
494         fillAction = LoadPtr;
495     } else if (registerFormat == DataFormatInt52) {
496         if (node->hasConstant())
497             fillAction = SetInt52Constant;
498         else if (info.spillFormat() == DataFormatInt52)
499             fillAction = Load64;
500         else if (info.spillFormat() == DataFormatStrictInt52)
501             fillAction = Load64ShiftInt52Left;
502         else if (info.spillFormat() == DataFormatNone)
503             fillAction = Load64;
504         else {
505             RELEASE_ASSERT_NOT_REACHED();
506 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
507             fillAction = Load64; // Make GCC happy.
508 #endif
509         }
510     } else if (registerFormat == DataFormatStrictInt52) {
511         if (node->hasConstant())
512             fillAction = SetStrictInt52Constant;
513         else if (info.spillFormat() == DataFormatInt52)
514             fillAction = Load64ShiftInt52Right;
515         else if (info.spillFormat() == DataFormatStrictInt52)
516             fillAction = Load64;
517         else if (info.spillFormat() == DataFormatNone)
518             fillAction = Load64;
519         else {
520             RELEASE_ASSERT_NOT_REACHED();
521 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
522             fillAction = Load64; // Make GCC happy.
523 #endif
524         }
525     } else {
526         ASSERT(registerFormat & DataFormatJS);
527 #if USE(JSVALUE64)
528         ASSERT(info.gpr() == source);
529         if (node->hasConstant()) {
530             if (node->isCellConstant())
531                 fillAction = SetTrustedJSConstant;
532             else
533                 fillAction = SetJSConstant;
534         } else if (info.spillFormat() == DataFormatInt32) {
535             ASSERT(registerFormat == DataFormatJSInt32);
536             fillAction = Load32PayloadBoxInt;
537         } else
538             fillAction = Load64;
539 #else
540         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
541         if (node->hasConstant())
542             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
543         else if (info.payloadGPR() == source)
544             fillAction = Load32Payload;
545         else { // Fill the Tag
546             switch (info.spillFormat()) {
547             case DataFormatInt32:
548                 ASSERT(registerFormat == DataFormatJSInt32);
549                 fillAction = SetInt32Tag;
550                 break;
551             case DataFormatCell:
552                 ASSERT(registerFormat == DataFormatJSCell);
553                 fillAction = SetCellTag;
554                 break;
555             case DataFormatBoolean:
556                 ASSERT(registerFormat == DataFormatJSBoolean);
557                 fillAction = SetBooleanTag;
558                 break;
559             default:
560                 fillAction = Load32Tag;
561                 break;
562             }
563         }
564 #endif
565     }
566         
567     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
568 }
569     
570 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
571 {
572     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
573     Node* node = info.node();
574     ASSERT(info.registerFormat() == DataFormatDouble);
575
576     SilentSpillAction spillAction;
577     SilentFillAction fillAction;
578         
579     if (!info.needsSpill())
580         spillAction = DoNothingForSpill;
581     else {
582         ASSERT(!node->hasConstant());
583         ASSERT(info.spillFormat() == DataFormatNone);
584         ASSERT(info.fpr() == source);
585         spillAction = StoreDouble;
586     }
587         
588 #if USE(JSVALUE64)
589     if (node->hasConstant()) {
590         node->asNumber(); // To get the assertion.
591         fillAction = SetDoubleConstant;
592     } else {
593         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
594         fillAction = LoadDouble;
595     }
596 #elif USE(JSVALUE32_64)
597     ASSERT(info.registerFormat() == DataFormatDouble);
598     if (node->hasConstant()) {
599         node->asNumber(); // To get the assertion.
600         fillAction = SetDoubleConstant;
601     } else
602         fillAction = LoadDouble;
603 #endif
604
605     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
606 }
607     
608 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
609 {
610     switch (plan.spillAction()) {
611     case DoNothingForSpill:
612         break;
613     case Store32Tag:
614         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
615         break;
616     case Store32Payload:
617         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
618         break;
619     case StorePtr:
620         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
621         break;
622 #if USE(JSVALUE64)
623     case Store64:
624         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
625         break;
626 #endif
627     case StoreDouble:
628         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
629         break;
630     default:
631         RELEASE_ASSERT_NOT_REACHED();
632     }
633 }
634     
635 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
636 {
637     switch (plan.fillAction()) {
638     case DoNothingForFill:
639         break;
640     case SetInt32Constant:
641         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
642         break;
643 #if USE(JSVALUE64)
644     case SetInt52Constant:
645         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
646         break;
647     case SetStrictInt52Constant:
648         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
649         break;
650 #endif // USE(JSVALUE64)
651     case SetBooleanConstant:
652         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
653         break;
654     case SetCellConstant:
655         ASSERT(plan.node()->constant()->value().isCell());
656         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
657         break;
658 #if USE(JSVALUE64)
659     case SetTrustedJSConstant:
660         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
661         break;
662     case SetJSConstant:
663         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
664         break;
665     case SetDoubleConstant:
666         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
667         break;
668     case Load32PayloadBoxInt:
669         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
670         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
671         break;
672     case Load32PayloadConvertToInt52:
673         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
674         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
675         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
676         break;
677     case Load32PayloadSignExtend:
678         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
679         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
680         break;
681 #else
682     case SetJSConstantTag:
683         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
684         break;
685     case SetJSConstantPayload:
686         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
687         break;
688     case SetInt32Tag:
689         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
690         break;
691     case SetCellTag:
692         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
693         break;
694     case SetBooleanTag:
695         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
696         break;
697     case SetDoubleConstant:
698         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
699         break;
700 #endif
701     case Load32Tag:
702         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
703         break;
704     case Load32Payload:
705         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
706         break;
707     case LoadPtr:
708         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
709         break;
710 #if USE(JSVALUE64)
711     case Load64:
712         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
713         break;
714     case Load64ShiftInt52Right:
715         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
716         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
717         break;
718     case Load64ShiftInt52Left:
719         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
720         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
721         break;
722 #endif
723     case LoadDouble:
724         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
725         break;
726     default:
727         RELEASE_ASSERT_NOT_REACHED();
728     }
729 }
730
731 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
732 {
733     JITCompiler::JumpList result;
734
735     IndexingType indexingModeMask = IsArray | IndexingShapeMask;
736     if (arrayMode.action() == Array::Write)
737         indexingModeMask |= CopyOnWrite;
738
739     switch (arrayMode.type()) {
740     case Array::Int32:
741     case Array::Double:
742     case Array::Contiguous:
743     case Array::Undecided:
744     case Array::ArrayStorage: {
745         IndexingType shape = arrayMode.shapeMask();
746         switch (arrayMode.arrayClass()) {
747         case Array::OriginalArray:
748         case Array::OriginalCopyOnWriteArray:
749             RELEASE_ASSERT_NOT_REACHED();
750             return result;
751
752         case Array::Array:
753             m_jit.and32(TrustedImm32(indexingModeMask), tempGPR);
754             result.append(m_jit.branch32(
755                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
756             return result;
757
758         case Array::NonArray:
759         case Array::OriginalNonArray:
760             m_jit.and32(TrustedImm32(indexingModeMask), tempGPR);
761             result.append(m_jit.branch32(
762                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
763             return result;
764
765         case Array::PossiblyArray:
766             m_jit.and32(TrustedImm32(indexingModeMask & ~IsArray), tempGPR);
767             result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
768             return result;
769         }
770
771         RELEASE_ASSERT_NOT_REACHED();
772         return result;
773     }
774
775     case Array::SlowPutArrayStorage: {
776         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
777
778         switch (arrayMode.arrayClass()) {
779         case Array::OriginalArray:
780         case Array::OriginalCopyOnWriteArray:
781             RELEASE_ASSERT_NOT_REACHED();
782             return result;
783
784         case Array::Array:
785             result.append(
786                 m_jit.branchTest32(
787                     MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
788             break;
789
790         case Array::NonArray:
791         case Array::OriginalNonArray:
792             result.append(
793                 m_jit.branchTest32(
794                     MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
795             break;
796
797         case Array::PossiblyArray:
798             break;
799         }
800
801         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
802         m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
803         result.append(
804             m_jit.branch32(
805                 MacroAssembler::Above, tempGPR,
806                 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
807         return result;
808     }
809     default:
810         CRASH();
811         break;
812     }
813     
814     return result;
815 }
816
817 void SpeculativeJIT::checkArray(Node* node)
818 {
819     ASSERT(node->arrayMode().isSpecific());
820     ASSERT(!node->arrayMode().doesConversion());
821     
822     SpeculateCellOperand base(this, node->child1());
823     GPRReg baseReg = base.gpr();
824     
825     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
826         noResult(m_currentNode);
827         return;
828     }
829     
830     switch (node->arrayMode().type()) {
831     case Array::AnyTypedArray:
832     case Array::String:
833         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
834         return;
835     case Array::Int32:
836     case Array::Double:
837     case Array::Contiguous:
838     case Array::Undecided:
839     case Array::ArrayStorage:
840     case Array::SlowPutArrayStorage: {
841         GPRTemporary temp(this);
842         GPRReg tempGPR = temp.gpr();
843         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
844         speculationCheck(
845             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
846             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
847         
848         noResult(m_currentNode);
849         return;
850     }
851     case Array::DirectArguments:
852         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
853         noResult(m_currentNode);
854         return;
855     case Array::ScopedArguments:
856         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
857         noResult(m_currentNode);
858         return;
859     default:
860         speculateCellTypeWithoutTypeFiltering(
861             node->child1(), baseReg,
862             typeForTypedArrayType(node->arrayMode().typedArrayType()));
863         noResult(m_currentNode);
864         return;
865     }
866 }
867
868 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
869 {
870     ASSERT(node->arrayMode().doesConversion());
871     
872     GPRTemporary temp(this);
873     GPRTemporary structure;
874     GPRReg tempGPR = temp.gpr();
875     GPRReg structureGPR = InvalidGPRReg;
876     
877     if (node->op() != ArrayifyToStructure) {
878         GPRTemporary realStructure(this);
879         structure.adopt(realStructure);
880         structureGPR = structure.gpr();
881     }
882         
883     // We can skip all that comes next if we already have array storage.
884     MacroAssembler::JumpList slowPath;
885     
886     if (node->op() == ArrayifyToStructure) {
887         ASSERT(!isCopyOnWrite(node->structure()->indexingMode()));
888         ASSERT((node->structure()->indexingType() & IndexingShapeMask) == node->arrayMode().shapeMask());
889         slowPath.append(m_jit.branchWeakStructure(
890             JITCompiler::NotEqual,
891             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
892             node->structure()));
893     } else {
894         m_jit.load8(
895             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
896         
897         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
898     }
899     
900     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
901         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
902     
903     noResult(m_currentNode);
904 }
905
906 void SpeculativeJIT::arrayify(Node* node)
907 {
908     ASSERT(node->arrayMode().isSpecific());
909     
910     SpeculateCellOperand base(this, node->child1());
911     
912     if (!node->child2()) {
913         arrayify(node, base.gpr(), InvalidGPRReg);
914         return;
915     }
916     
917     SpeculateInt32Operand property(this, node->child2());
918     
919     arrayify(node, base.gpr(), property.gpr());
920 }
921
922 GPRReg SpeculativeJIT::fillStorage(Edge edge)
923 {
924     VirtualRegister virtualRegister = edge->virtualRegister();
925     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
926     
927     switch (info.registerFormat()) {
928     case DataFormatNone: {
929         if (info.spillFormat() == DataFormatStorage) {
930             GPRReg gpr = allocate();
931             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
932             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
933             info.fillStorage(*m_stream, gpr);
934             return gpr;
935         }
936         
937         // Must be a cell; fill it as a cell and then return the pointer.
938         return fillSpeculateCell(edge);
939     }
940         
941     case DataFormatStorage: {
942         GPRReg gpr = info.gpr();
943         m_gprs.lock(gpr);
944         return gpr;
945     }
946         
947     default:
948         return fillSpeculateCell(edge);
949     }
950 }
951
952 void SpeculativeJIT::useChildren(Node* node)
953 {
954     if (node->flags() & NodeHasVarArgs) {
955         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
956             if (!!m_jit.graph().m_varArgChildren[childIdx])
957                 use(m_jit.graph().m_varArgChildren[childIdx]);
958         }
959     } else {
960         Edge child1 = node->child1();
961         if (!child1) {
962             ASSERT(!node->child2() && !node->child3());
963             return;
964         }
965         use(child1);
966         
967         Edge child2 = node->child2();
968         if (!child2) {
969             ASSERT(!node->child3());
970             return;
971         }
972         use(child2);
973         
974         Edge child3 = node->child3();
975         if (!child3)
976             return;
977         use(child3);
978     }
979 }
980
981 void SpeculativeJIT::compileGetById(Node* node, AccessType accessType)
982 {
983     ASSERT(accessType == AccessType::Get || accessType == AccessType::GetDirect || accessType == AccessType::TryGet);
984
985     switch (node->child1().useKind()) {
986     case CellUse: {
987         SpeculateCellOperand base(this, node->child1());
988         JSValueRegsTemporary result(this, Reuse, base);
989
990         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
991         JSValueRegs resultRegs = result.regs();
992
993         base.use();
994
995         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, accessType);
996
997         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
998         break;
999     }
1000
1001     case UntypedUse: {
1002         JSValueOperand base(this, node->child1());
1003         JSValueRegsTemporary result(this, Reuse, base);
1004
1005         JSValueRegs baseRegs = base.jsValueRegs();
1006         JSValueRegs resultRegs = result.regs();
1007
1008         base.use();
1009
1010         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1011
1012         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, accessType);
1013
1014         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1015         break;
1016     }
1017
1018     default:
1019         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1020         break;
1021     }
1022 }
1023
1024 void SpeculativeJIT::compileGetByIdFlush(Node* node, AccessType accessType)
1025 {
1026     switch (node->child1().useKind()) {
1027     case CellUse: {
1028         SpeculateCellOperand base(this, node->child1());
1029         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
1030
1031         JSValueRegsFlushedCallResult result(this);
1032         JSValueRegs resultRegs = result.regs();
1033
1034         base.use();
1035
1036         flushRegisters();
1037
1038         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), DontSpill, accessType);
1039
1040         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1041         break;
1042     }
1043
1044     case UntypedUse: {
1045         JSValueOperand base(this, node->child1());
1046         JSValueRegs baseRegs = base.jsValueRegs();
1047
1048         JSValueRegsFlushedCallResult result(this);
1049         JSValueRegs resultRegs = result.regs();
1050
1051         base.use();
1052
1053         flushRegisters();
1054
1055         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1056
1057         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, DontSpill, accessType);
1058
1059         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1060         break;
1061     }
1062
1063     default:
1064         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1065         break;
1066     }
1067 }
1068
1069 void SpeculativeJIT::compileInById(Node* node)
1070 {
1071     SpeculateCellOperand base(this, node->child1());
1072     JSValueRegsTemporary result(this, Reuse, base, PayloadWord);
1073
1074     GPRReg baseGPR = base.gpr();
1075     JSValueRegs resultRegs = result.regs();
1076
1077     base.use();
1078
1079     CodeOrigin codeOrigin = node->origin.semantic;
1080     CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size());
1081     RegisterSet usedRegisters = this->usedRegisters();
1082     JITInByIdGenerator gen(
1083         m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, identifierUID(node->identifierNumber()),
1084         JSValueRegs::payloadOnly(baseGPR), resultRegs);
1085     gen.generateFastPath(m_jit);
1086
1087     auto slowPath = slowPathCall(
1088         gen.slowPathJump(), this, operationInByIdOptimize,
1089         NeedToSpill, ExceptionCheckRequirement::CheckNeeded,
1090         resultRegs, gen.stubInfo(), CCallHelpers::CellValue(baseGPR), identifierUID(node->identifierNumber()));
1091
1092     m_jit.addInById(gen, slowPath.get());
1093     addSlowPathGenerator(WTFMove(slowPath));
1094
1095     blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1096 }
1097
1098 void SpeculativeJIT::compileInByVal(Node* node)
1099 {
1100     SpeculateCellOperand base(this, node->child1());
1101     JSValueOperand key(this, node->child2());
1102
1103     GPRReg baseGPR = base.gpr();
1104     JSValueRegs regs = key.jsValueRegs();
1105
1106     base.use();
1107     key.use();
1108
1109     flushRegisters();
1110     JSValueRegsFlushedCallResult result(this);
1111     JSValueRegs resultRegs = result.regs();
1112     callOperation(operationInByVal, resultRegs, baseGPR, regs);
1113     m_jit.exceptionCheck();
1114     blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1115 }
1116
1117 void SpeculativeJIT::compileDeleteById(Node* node)
1118 {
1119     JSValueOperand value(this, node->child1());
1120     GPRFlushedCallResult result(this);
1121
1122     JSValueRegs valueRegs = value.jsValueRegs();
1123     GPRReg resultGPR = result.gpr();
1124
1125     value.use();
1126
1127     flushRegisters();
1128     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1129     m_jit.exceptionCheck();
1130
1131     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1132 }
1133
1134 void SpeculativeJIT::compileDeleteByVal(Node* node)
1135 {
1136     JSValueOperand base(this, node->child1());
1137     JSValueOperand key(this, node->child2());
1138     GPRFlushedCallResult result(this);
1139
1140     JSValueRegs baseRegs = base.jsValueRegs();
1141     JSValueRegs keyRegs = key.jsValueRegs();
1142     GPRReg resultGPR = result.gpr();
1143
1144     base.use();
1145     key.use();
1146
1147     flushRegisters();
1148     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1149     m_jit.exceptionCheck();
1150
1151     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1152 }
1153
1154 void SpeculativeJIT::compilePushWithScope(Node* node)
1155 {
1156     SpeculateCellOperand currentScope(this, node->child1());
1157     GPRReg currentScopeGPR = currentScope.gpr();
1158
1159     GPRFlushedCallResult result(this);
1160     GPRReg resultGPR = result.gpr();
1161
1162     auto objectEdge = node->child2();
1163     if (objectEdge.useKind() == ObjectUse) {
1164         SpeculateCellOperand object(this, objectEdge);
1165         GPRReg objectGPR = object.gpr();
1166         speculateObject(objectEdge, objectGPR);
1167
1168         flushRegisters();
1169         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1170         // No exception check here as we did not have to call toObject().
1171     } else {
1172         ASSERT(objectEdge.useKind() == UntypedUse);
1173         JSValueOperand object(this, objectEdge);
1174         JSValueRegs objectRegs = object.jsValueRegs();
1175
1176         flushRegisters();
1177         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1178         m_jit.exceptionCheck();
1179     }
1180     
1181     cellResult(resultGPR, node);
1182 }
1183
1184 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1185 {
1186     unsigned branchIndexInBlock = detectPeepHoleBranch();
1187     if (branchIndexInBlock != UINT_MAX) {
1188         Node* branchNode = m_block->at(branchIndexInBlock);
1189
1190         ASSERT(node->adjustedRefCount() == 1);
1191         
1192         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1193     
1194         m_indexInBlock = branchIndexInBlock;
1195         m_currentNode = branchNode;
1196         
1197         return true;
1198     }
1199     
1200     nonSpeculativeNonPeepholeStrictEq(node, invert);
1201     
1202     return false;
1203 }
1204
1205 static const char* dataFormatString(DataFormat format)
1206 {
1207     // These values correspond to the DataFormat enum.
1208     const char* strings[] = {
1209         "[  ]",
1210         "[ i]",
1211         "[ d]",
1212         "[ c]",
1213         "Err!",
1214         "Err!",
1215         "Err!",
1216         "Err!",
1217         "[J ]",
1218         "[Ji]",
1219         "[Jd]",
1220         "[Jc]",
1221         "Err!",
1222         "Err!",
1223         "Err!",
1224         "Err!",
1225     };
1226     return strings[format];
1227 }
1228
1229 void SpeculativeJIT::dump(const char* label)
1230 {
1231     if (label)
1232         dataLogF("<%s>\n", label);
1233
1234     dataLogF("  gprs:\n");
1235     m_gprs.dump();
1236     dataLogF("  fprs:\n");
1237     m_fprs.dump();
1238     dataLogF("  VirtualRegisters:\n");
1239     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1240         GenerationInfo& info = m_generationInfo[i];
1241         if (info.alive())
1242             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1243         else
1244             dataLogF("    % 3d:[__][__]", i);
1245         if (info.registerFormat() == DataFormatDouble)
1246             dataLogF(":fpr%d\n", info.fpr());
1247         else if (info.registerFormat() != DataFormatNone
1248 #if USE(JSVALUE32_64)
1249             && !(info.registerFormat() & DataFormatJS)
1250 #endif
1251             ) {
1252             ASSERT(info.gpr() != InvalidGPRReg);
1253             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1254         } else
1255             dataLogF("\n");
1256     }
1257     if (label)
1258         dataLogF("</%s>\n", label);
1259 }
1260
1261 GPRTemporary::GPRTemporary()
1262     : m_jit(0)
1263     , m_gpr(InvalidGPRReg)
1264 {
1265 }
1266
1267 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1268     : m_jit(jit)
1269     , m_gpr(InvalidGPRReg)
1270 {
1271     m_gpr = m_jit->allocate();
1272 }
1273
1274 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1275     : m_jit(jit)
1276     , m_gpr(InvalidGPRReg)
1277 {
1278     m_gpr = m_jit->allocate(specific);
1279 }
1280
1281 #if USE(JSVALUE32_64)
1282 GPRTemporary::GPRTemporary(
1283     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1284     : m_jit(jit)
1285     , m_gpr(InvalidGPRReg)
1286 {
1287     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1288         m_gpr = m_jit->reuse(op1.gpr(which));
1289     else
1290         m_gpr = m_jit->allocate();
1291 }
1292 #else // USE(JSVALUE32_64)
1293 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord)
1294     : GPRTemporary(jit, Reuse, op1)
1295 {
1296 }
1297 #endif
1298
1299 JSValueRegsTemporary::JSValueRegsTemporary() { }
1300
1301 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1302 #if USE(JSVALUE64)
1303     : m_gpr(jit)
1304 #else
1305     : m_payloadGPR(jit)
1306     , m_tagGPR(jit)
1307 #endif
1308 {
1309 }
1310
1311 #if USE(JSVALUE64)
1312 template<typename T>
1313 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1314     : m_gpr(jit, Reuse, operand)
1315 {
1316 }
1317 #else
1318 template<typename T>
1319 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1320 {
1321     if (resultWord == PayloadWord) {
1322         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1323         m_tagGPR = GPRTemporary(jit);
1324     } else {
1325         m_payloadGPR = GPRTemporary(jit);
1326         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1327     }
1328 }
1329 #endif
1330
1331 #if USE(JSVALUE64)
1332 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1333 {
1334     m_gpr = GPRTemporary(jit, Reuse, operand);
1335 }
1336 #else
1337 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1338 {
1339     if (jit->canReuse(operand.node())) {
1340         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1341         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1342     } else {
1343         m_payloadGPR = GPRTemporary(jit);
1344         m_tagGPR = GPRTemporary(jit);
1345     }
1346 }
1347 #endif
1348
1349 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1350
1351 JSValueRegs JSValueRegsTemporary::regs()
1352 {
1353 #if USE(JSVALUE64)
1354     return JSValueRegs(m_gpr.gpr());
1355 #else
1356     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1357 #endif
1358 }
1359
1360 void GPRTemporary::adopt(GPRTemporary& other)
1361 {
1362     ASSERT(!m_jit);
1363     ASSERT(m_gpr == InvalidGPRReg);
1364     ASSERT(other.m_jit);
1365     ASSERT(other.m_gpr != InvalidGPRReg);
1366     m_jit = other.m_jit;
1367     m_gpr = other.m_gpr;
1368     other.m_jit = 0;
1369     other.m_gpr = InvalidGPRReg;
1370 }
1371
1372 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1373 {
1374     ASSERT(other.m_jit);
1375     ASSERT(other.m_fpr != InvalidFPRReg);
1376     m_jit = other.m_jit;
1377     m_fpr = other.m_fpr;
1378
1379     other.m_jit = nullptr;
1380 }
1381
1382 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1383     : m_jit(jit)
1384     , m_fpr(InvalidFPRReg)
1385 {
1386     m_fpr = m_jit->fprAllocate();
1387 }
1388
1389 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1390     : m_jit(jit)
1391     , m_fpr(InvalidFPRReg)
1392 {
1393     if (m_jit->canReuse(op1.node()))
1394         m_fpr = m_jit->reuse(op1.fpr());
1395     else
1396         m_fpr = m_jit->fprAllocate();
1397 }
1398
1399 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1400     : m_jit(jit)
1401     , m_fpr(InvalidFPRReg)
1402 {
1403     if (m_jit->canReuse(op1.node()))
1404         m_fpr = m_jit->reuse(op1.fpr());
1405     else if (m_jit->canReuse(op2.node()))
1406         m_fpr = m_jit->reuse(op2.fpr());
1407     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1408         m_fpr = m_jit->reuse(op1.fpr());
1409     else
1410         m_fpr = m_jit->fprAllocate();
1411 }
1412
1413 #if USE(JSVALUE32_64)
1414 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1415     : m_jit(jit)
1416     , m_fpr(InvalidFPRReg)
1417 {
1418     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1419         m_fpr = m_jit->reuse(op1.fpr());
1420     else
1421         m_fpr = m_jit->fprAllocate();
1422 }
1423 #endif
1424
1425 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1426 {
1427     BasicBlock* taken = branchNode->branchData()->taken.block;
1428     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1429
1430     if (taken == nextBlock()) {
1431         condition = MacroAssembler::invert(condition);
1432         std::swap(taken, notTaken);
1433     }
1434
1435     SpeculateDoubleOperand op1(this, node->child1());
1436     SpeculateDoubleOperand op2(this, node->child2());
1437     
1438     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1439     jump(notTaken);
1440 }
1441
1442 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1443 {
1444     BasicBlock* taken = branchNode->branchData()->taken.block;
1445     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1446
1447     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1448     
1449     if (taken == nextBlock()) {
1450         condition = MacroAssembler::NotEqual;
1451         BasicBlock* tmp = taken;
1452         taken = notTaken;
1453         notTaken = tmp;
1454     }
1455
1456     SpeculateCellOperand op1(this, node->child1());
1457     SpeculateCellOperand op2(this, node->child2());
1458     
1459     GPRReg op1GPR = op1.gpr();
1460     GPRReg op2GPR = op2.gpr();
1461     
1462     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1463         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1464             speculationCheck(
1465                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1466         }
1467         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1468             speculationCheck(
1469                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1470         }
1471     } else {
1472         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1473             speculationCheck(
1474                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1475                 m_jit.branchIfNotObject(op1GPR));
1476         }
1477         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1478             m_jit.branchTest8(
1479                 MacroAssembler::NonZero, 
1480                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1481                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1482
1483         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1484             speculationCheck(
1485                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1486                 m_jit.branchIfNotObject(op2GPR));
1487         }
1488         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1489             m_jit.branchTest8(
1490                 MacroAssembler::NonZero, 
1491                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1492                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1493     }
1494
1495     branchPtr(condition, op1GPR, op2GPR, taken);
1496     jump(notTaken);
1497 }
1498
1499 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1500 {
1501     BasicBlock* taken = branchNode->branchData()->taken.block;
1502     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1503
1504     // The branch instruction will branch to the taken block.
1505     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1506     if (taken == nextBlock()) {
1507         condition = JITCompiler::invert(condition);
1508         BasicBlock* tmp = taken;
1509         taken = notTaken;
1510         notTaken = tmp;
1511     }
1512
1513     if (node->child1()->isInt32Constant()) {
1514         int32_t imm = node->child1()->asInt32();
1515         SpeculateBooleanOperand op2(this, node->child2());
1516         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1517     } else if (node->child2()->isInt32Constant()) {
1518         SpeculateBooleanOperand op1(this, node->child1());
1519         int32_t imm = node->child2()->asInt32();
1520         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1521     } else {
1522         SpeculateBooleanOperand op1(this, node->child1());
1523         SpeculateBooleanOperand op2(this, node->child2());
1524         branch32(condition, op1.gpr(), op2.gpr(), taken);
1525     }
1526
1527     jump(notTaken);
1528 }
1529
1530 void SpeculativeJIT::compileStringSlice(Node* node)
1531 {
1532     SpeculateCellOperand string(this, node->child1());
1533     GPRTemporary startIndex(this);
1534     GPRTemporary temp(this);
1535     GPRTemporary temp2(this);
1536
1537     GPRReg stringGPR = string.gpr();
1538     GPRReg startIndexGPR = startIndex.gpr();
1539     GPRReg tempGPR = temp.gpr();
1540     GPRReg temp2GPR = temp2.gpr();
1541
1542     speculateString(node->child1(), stringGPR);
1543
1544     {
1545         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1546
1547         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1548         if (node->child3())
1549             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1550         else
1551             m_jit.move(temp2GPR, tempGPR);
1552     }
1553
1554     CCallHelpers::JumpList doneCases;
1555     CCallHelpers::JumpList slowCases;
1556
1557     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1558     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1559     doneCases.append(m_jit.jump());
1560
1561     nonEmptyCase.link(&m_jit);
1562     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1563     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1564
1565     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1566     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1567
1568     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1569
1570     // Load the character into scratchReg
1571     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1572     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1573
1574     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1575     auto cont8Bit = m_jit.jump();
1576
1577     is16Bit.link(&m_jit);
1578     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1579
1580     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1581
1582     // 8 bit string values don't need the isASCII check.
1583     cont8Bit.link(&m_jit);
1584
1585     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1586     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1587     m_jit.loadPtr(tempGPR, tempGPR);
1588
1589     addSlowPathGenerator(
1590         slowPathCall(
1591             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1592
1593     addSlowPathGenerator(
1594         slowPathCall(
1595             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1596
1597     doneCases.link(&m_jit);
1598     cellResult(tempGPR, node);
1599 }
1600
1601 void SpeculativeJIT::compileToLowerCase(Node* node)
1602 {
1603     ASSERT(node->op() == ToLowerCase);
1604     SpeculateCellOperand string(this, node->child1());
1605     GPRTemporary temp(this);
1606     GPRTemporary index(this);
1607     GPRTemporary charReg(this);
1608     GPRTemporary length(this);
1609
1610     GPRReg stringGPR = string.gpr();
1611     GPRReg tempGPR = temp.gpr();
1612     GPRReg indexGPR = index.gpr();
1613     GPRReg charGPR = charReg.gpr();
1614     GPRReg lengthGPR = length.gpr();
1615
1616     speculateString(node->child1(), stringGPR);
1617
1618     CCallHelpers::JumpList slowPath;
1619
1620     m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1621
1622     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1623     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1624
1625     slowPath.append(m_jit.branchTest32(
1626         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1627         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1628     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1629     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1630
1631     auto loopStart = m_jit.label();
1632     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1633     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1634     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1635     m_jit.sub32(TrustedImm32('A'), charGPR);
1636     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1637
1638     m_jit.add32(TrustedImm32(1), indexGPR);
1639     m_jit.jump().linkTo(loopStart, &m_jit);
1640     
1641     slowPath.link(&m_jit);
1642     silentSpillAllRegisters(lengthGPR);
1643     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1644     silentFillAllRegisters();
1645     m_jit.exceptionCheck();
1646     auto done = m_jit.jump();
1647
1648     loopDone.link(&m_jit);
1649     m_jit.move(stringGPR, lengthGPR);
1650
1651     done.link(&m_jit);
1652     cellResult(lengthGPR, node);
1653 }
1654
1655 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1656 {
1657     BasicBlock* taken = branchNode->branchData()->taken.block;
1658     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1659
1660     // The branch instruction will branch to the taken block.
1661     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1662     if (taken == nextBlock()) {
1663         condition = JITCompiler::invert(condition);
1664         BasicBlock* tmp = taken;
1665         taken = notTaken;
1666         notTaken = tmp;
1667     }
1668
1669     if (node->child1()->isInt32Constant()) {
1670         int32_t imm = node->child1()->asInt32();
1671         SpeculateInt32Operand op2(this, node->child2());
1672         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1673     } else if (node->child2()->isInt32Constant()) {
1674         SpeculateInt32Operand op1(this, node->child1());
1675         int32_t imm = node->child2()->asInt32();
1676         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1677     } else {
1678         SpeculateInt32Operand op1(this, node->child1());
1679         SpeculateInt32Operand op2(this, node->child2());
1680         branch32(condition, op1.gpr(), op2.gpr(), taken);
1681     }
1682
1683     jump(notTaken);
1684 }
1685
1686 // Returns true if the compare is fused with a subsequent branch.
1687 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1688 {
1689     // Fused compare & branch.
1690     unsigned branchIndexInBlock = detectPeepHoleBranch();
1691     if (branchIndexInBlock != UINT_MAX) {
1692         Node* branchNode = m_block->at(branchIndexInBlock);
1693
1694         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1695         // so can be no intervening nodes to also reference the compare. 
1696         ASSERT(node->adjustedRefCount() == 1);
1697
1698         if (node->isBinaryUseKind(Int32Use))
1699             compilePeepHoleInt32Branch(node, branchNode, condition);
1700 #if USE(JSVALUE64)
1701         else if (node->isBinaryUseKind(Int52RepUse))
1702             compilePeepHoleInt52Branch(node, branchNode, condition);
1703 #endif // USE(JSVALUE64)
1704         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1705             // Use non-peephole comparison, for now.
1706             return false;
1707         } else if (node->isBinaryUseKind(DoubleRepUse))
1708             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1709         else if (node->op() == CompareEq) {
1710             if (node->isBinaryUseKind(BooleanUse))
1711                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1712             else if (node->isBinaryUseKind(SymbolUse))
1713                 compilePeepHoleSymbolEquality(node, branchNode);
1714             else if (node->isBinaryUseKind(ObjectUse))
1715                 compilePeepHoleObjectEquality(node, branchNode);
1716             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1717                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1718             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1719                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1720             else if (!needsTypeCheck(node->child1(), SpecOther))
1721                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1722             else if (!needsTypeCheck(node->child2(), SpecOther))
1723                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1724             else {
1725                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1726                 return true;
1727             }
1728         } else {
1729             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1730             return true;
1731         }
1732
1733         use(node->child1());
1734         use(node->child2());
1735         m_indexInBlock = branchIndexInBlock;
1736         m_currentNode = branchNode;
1737         return true;
1738     }
1739     return false;
1740 }
1741
1742 void SpeculativeJIT::noticeOSRBirth(Node* node)
1743 {
1744     if (!node->hasVirtualRegister())
1745         return;
1746     
1747     VirtualRegister virtualRegister = node->virtualRegister();
1748     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1749     
1750     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1751 }
1752
1753 void SpeculativeJIT::compileMovHint(Node* node)
1754 {
1755     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1756     
1757     Node* child = node->child1().node();
1758     noticeOSRBirth(child);
1759     
1760     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1761 }
1762
1763 void SpeculativeJIT::bail(AbortReason reason)
1764 {
1765     if (verboseCompilationEnabled())
1766         dataLog("Bailing compilation.\n");
1767     m_compileOkay = true;
1768     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1769     clearGenerationInfo();
1770 }
1771
1772 void SpeculativeJIT::compileCurrentBlock()
1773 {
1774     ASSERT(m_compileOkay);
1775     
1776     if (!m_block)
1777         return;
1778     
1779     ASSERT(m_block->isReachable);
1780     
1781     m_jit.blockHeads()[m_block->index] = m_jit.label();
1782
1783     if (!m_block->intersectionOfCFAHasVisited) {
1784         // Don't generate code for basic blocks that are unreachable according to CFA.
1785         // But to be sure that nobody has generated a jump to this block, drop in a
1786         // breakpoint here.
1787         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1788         return;
1789     }
1790
1791     if (m_block->isCatchEntrypoint) {
1792         m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1793         if (Options::zeroStackFrame())
1794             m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1795         m_jit.emitSaveCalleeSaves();
1796         m_jit.emitMaterializeTagCheckRegisters();
1797         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1798     }
1799
1800     m_stream->appendAndLog(VariableEvent::reset());
1801     
1802     m_jit.jitAssertHasValidCallFrame();
1803     m_jit.jitAssertTagsInPlace();
1804     m_jit.jitAssertArgumentCountSane();
1805
1806     m_state.reset();
1807     m_state.beginBasicBlock(m_block);
1808     
1809     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1810         int operand = m_block->variablesAtHead.operandForIndex(i);
1811         Node* node = m_block->variablesAtHead[i];
1812         if (!node)
1813             continue; // No need to record dead SetLocal's.
1814         
1815         VariableAccessData* variable = node->variableAccessData();
1816         DataFormat format;
1817         if (!node->refCount())
1818             continue; // No need to record dead SetLocal's.
1819         format = dataFormatFor(variable->flushFormat());
1820         m_stream->appendAndLog(
1821             VariableEvent::setLocal(
1822                 VirtualRegister(operand),
1823                 variable->machineLocal(),
1824                 format));
1825     }
1826
1827     m_origin = NodeOrigin();
1828     
1829     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1830         m_currentNode = m_block->at(m_indexInBlock);
1831         
1832         // We may have hit a contradiction that the CFA was aware of but that the JIT
1833         // didn't cause directly.
1834         if (!m_state.isValid()) {
1835             bail(DFGBailedAtTopOfBlock);
1836             return;
1837         }
1838
1839         m_interpreter.startExecuting();
1840         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1841         m_jit.setForNode(m_currentNode);
1842         m_origin = m_currentNode->origin;
1843         if (validationEnabled())
1844             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1845         m_lastGeneratedNode = m_currentNode->op();
1846         
1847         ASSERT(m_currentNode->shouldGenerate());
1848         
1849         if (verboseCompilationEnabled()) {
1850             dataLogF(
1851                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1852                 (int)m_currentNode->index(),
1853                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1854             dataLog("\n");
1855         }
1856
1857         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1858             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1859
1860         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1861
1862         compile(m_currentNode);
1863         
1864         if (belongsInMinifiedGraph(m_currentNode->op()))
1865             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1866         
1867 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1868         m_jit.clearRegisterAllocationOffsets();
1869 #endif
1870         
1871         if (!m_compileOkay) {
1872             bail(DFGBailedAtEndOfNode);
1873             return;
1874         }
1875         
1876         // Make sure that the abstract state is rematerialized for the next node.
1877         m_interpreter.executeEffects(m_indexInBlock);
1878     }
1879     
1880     // Perform the most basic verification that children have been used correctly.
1881     if (!ASSERT_DISABLED) {
1882         for (auto& info : m_generationInfo)
1883             RELEASE_ASSERT(!info.alive());
1884     }
1885 }
1886
1887 // If we are making type predictions about our arguments then
1888 // we need to check that they are correct on function entry.
1889 void SpeculativeJIT::checkArgumentTypes()
1890 {
1891     ASSERT(!m_currentNode);
1892     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1893
1894     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1895     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1896         Node* node = arguments[i];
1897         if (!node) {
1898             // The argument is dead. We don't do any checks for such arguments.
1899             continue;
1900         }
1901         
1902         ASSERT(node->op() == SetArgument);
1903         ASSERT(node->shouldGenerate());
1904
1905         VariableAccessData* variableAccessData = node->variableAccessData();
1906         FlushFormat format = variableAccessData->flushFormat();
1907         
1908         if (format == FlushedJSValue)
1909             continue;
1910         
1911         VirtualRegister virtualRegister = variableAccessData->local();
1912
1913         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1914         
1915 #if USE(JSVALUE64)
1916         switch (format) {
1917         case FlushedInt32: {
1918             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1919             break;
1920         }
1921         case FlushedBoolean: {
1922             GPRTemporary temp(this);
1923             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1924             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1925             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1926             break;
1927         }
1928         case FlushedCell: {
1929             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1930             break;
1931         }
1932         default:
1933             RELEASE_ASSERT_NOT_REACHED();
1934             break;
1935         }
1936 #else
1937         switch (format) {
1938         case FlushedInt32: {
1939             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1940             break;
1941         }
1942         case FlushedBoolean: {
1943             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1944             break;
1945         }
1946         case FlushedCell: {
1947             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1948             break;
1949         }
1950         default:
1951             RELEASE_ASSERT_NOT_REACHED();
1952             break;
1953         }
1954 #endif
1955     }
1956
1957     m_origin = NodeOrigin();
1958 }
1959
1960 bool SpeculativeJIT::compile()
1961 {
1962     checkArgumentTypes();
1963     
1964     ASSERT(!m_currentNode);
1965     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1966         m_jit.setForBlockIndex(blockIndex);
1967         m_block = m_jit.graph().block(blockIndex);
1968         compileCurrentBlock();
1969     }
1970     linkBranches();
1971     return true;
1972 }
1973
1974 void SpeculativeJIT::createOSREntries()
1975 {
1976     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1977         BasicBlock* block = m_jit.graph().block(blockIndex);
1978         if (!block)
1979             continue;
1980         if (block->isOSRTarget || block->isCatchEntrypoint) {
1981             // Currently we don't have OSR entry trampolines. We could add them
1982             // here if need be.
1983             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1984         }
1985     }
1986 }
1987
1988 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1989 {
1990     unsigned osrEntryIndex = 0;
1991     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1992         BasicBlock* block = m_jit.graph().block(blockIndex);
1993         if (!block)
1994             continue;
1995         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1996             continue;
1997         if (block->isCatchEntrypoint) {
1998             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1999             Vector<FlushFormat> argumentFormats;
2000             argumentFormats.reserveInitialCapacity(argumentsVector.size());
2001             for (Node* setArgument : argumentsVector) {
2002                 if (setArgument) {
2003                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
2004                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
2005                     argumentFormats.uncheckedAppend(flushFormat);
2006                 } else
2007                     argumentFormats.uncheckedAppend(DeadFlush);
2008             }
2009             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
2010         } else {
2011             ASSERT(block->isOSRTarget);
2012             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
2013         }
2014     }
2015
2016     m_jit.jitCode()->finalizeOSREntrypoints();
2017     m_jit.jitCode()->common.finalizeCatchEntrypoints();
2018
2019     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
2020     
2021     if (verboseCompilationEnabled()) {
2022         DumpContext dumpContext;
2023         dataLog("OSR Entries:\n");
2024         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
2025             dataLog("    ", inContext(entryData, &dumpContext), "\n");
2026         if (!dumpContext.isEmpty())
2027             dumpContext.dump(WTF::dataFile());
2028     }
2029 }
2030     
2031 void SpeculativeJIT::compileCheckTraps(Node*)
2032 {
2033     ASSERT(Options::usePollingTraps());
2034     GPRTemporary unused(this);
2035     GPRReg unusedGPR = unused.gpr();
2036
2037     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2038         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2039
2040     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2041 }
2042
2043 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2044 {
2045     Edge child3 = m_jit.graph().varArgChild(node, 2);
2046     Edge child4 = m_jit.graph().varArgChild(node, 3);
2047
2048     ArrayMode arrayMode = node->arrayMode();
2049     
2050     GPRReg baseReg = base.gpr();
2051     GPRReg propertyReg = property.gpr();
2052     
2053     SpeculateDoubleOperand value(this, child3);
2054
2055     FPRReg valueReg = value.fpr();
2056     
2057     DFG_TYPE_CHECK(
2058         JSValueRegs(), child3, SpecFullRealNumber,
2059         m_jit.branchDouble(
2060             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2061     
2062     if (!m_compileOkay)
2063         return;
2064     
2065     StorageOperand storage(this, child4);
2066     GPRReg storageReg = storage.gpr();
2067
2068     if (node->op() == PutByValAlias) {
2069         // Store the value to the array.
2070         GPRReg propertyReg = property.gpr();
2071         FPRReg valueReg = value.fpr();
2072         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2073         
2074         noResult(m_currentNode);
2075         return;
2076     }
2077     
2078     GPRTemporary temporary;
2079     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2080
2081     MacroAssembler::Jump slowCase;
2082     
2083     if (arrayMode.isInBounds()) {
2084         speculationCheck(
2085             OutOfBounds, JSValueRegs(), 0,
2086             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2087     } else {
2088         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2089         
2090         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2091         
2092         if (!arrayMode.isOutOfBounds())
2093             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2094         
2095         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2096         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2097
2098         inBounds.link(&m_jit);
2099     }
2100     
2101     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2102
2103     base.use();
2104     property.use();
2105     value.use();
2106     storage.use();
2107     
2108     if (arrayMode.isOutOfBounds()) {
2109         addSlowPathGenerator(
2110             slowPathCall(
2111                 slowCase, this,
2112                 m_jit.codeBlock()->isStrictMode()
2113                     ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2114                     : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2115                 NoResult, baseReg, propertyReg, valueReg));
2116     }
2117
2118     noResult(m_currentNode, UseChildrenCalledExplicitly);
2119 }
2120
2121 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2122 {
2123     SpeculateCellOperand string(this, node->child1());
2124     SpeculateStrictInt32Operand index(this, node->child2());
2125     StorageOperand storage(this, node->child3());
2126
2127     GPRReg stringReg = string.gpr();
2128     GPRReg indexReg = index.gpr();
2129     GPRReg storageReg = storage.gpr();
2130     
2131     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2132
2133     // unsigned comparison so we can filter out negative indices and indices that are too large
2134     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2135
2136     GPRTemporary scratch(this);
2137     GPRReg scratchReg = scratch.gpr();
2138
2139     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2140
2141     // Load the character into scratchReg
2142     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2143
2144     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2145     JITCompiler::Jump cont8Bit = m_jit.jump();
2146
2147     is16Bit.link(&m_jit);
2148
2149     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2150
2151     cont8Bit.link(&m_jit);
2152
2153     int32Result(scratchReg, m_currentNode);
2154 }
2155
2156 void SpeculativeJIT::compileGetByValOnString(Node* node)
2157 {
2158     SpeculateCellOperand base(this, m_graph.child(node, 0));
2159     SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2160     StorageOperand storage(this, m_graph.child(node, 2));
2161     GPRReg baseReg = base.gpr();
2162     GPRReg propertyReg = property.gpr();
2163     GPRReg storageReg = storage.gpr();
2164
2165     GPRTemporary scratch(this);
2166     GPRReg scratchReg = scratch.gpr();
2167 #if USE(JSVALUE32_64)
2168     GPRTemporary resultTag;
2169     GPRReg resultTagReg = InvalidGPRReg;
2170     if (node->arrayMode().isOutOfBounds()) {
2171         GPRTemporary realResultTag(this);
2172         resultTag.adopt(realResultTag);
2173         resultTagReg = resultTag.gpr();
2174     }
2175 #endif
2176
2177     ASSERT(ArrayMode(Array::String, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2178
2179     // unsigned comparison so we can filter out negative indices and indices that are too large
2180     JITCompiler::Jump outOfBounds = m_jit.branch32(
2181         MacroAssembler::AboveOrEqual, propertyReg,
2182         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2183     if (node->arrayMode().isInBounds())
2184         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2185
2186     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2187
2188     // Load the character into scratchReg
2189     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2190
2191     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2192     JITCompiler::Jump cont8Bit = m_jit.jump();
2193
2194     is16Bit.link(&m_jit);
2195
2196     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2197
2198     JITCompiler::Jump bigCharacter =
2199         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2200
2201     // 8 bit string values don't need the isASCII check.
2202     cont8Bit.link(&m_jit);
2203
2204     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2205     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2206     m_jit.loadPtr(scratchReg, scratchReg);
2207
2208     addSlowPathGenerator(
2209         slowPathCall(
2210             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2211
2212     if (node->arrayMode().isOutOfBounds()) {
2213 #if USE(JSVALUE32_64)
2214         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2215 #endif
2216
2217         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2218         bool prototypeChainIsSane = false;
2219         if (globalObject->stringPrototypeChainIsSane()) {
2220             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2221             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2222             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2223             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2224             // indexed properties either.
2225             // https://bugs.webkit.org/show_bug.cgi?id=144668
2226             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure(*m_jit.vm()));
2227             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure(*m_jit.vm()));
2228             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2229         }
2230         if (prototypeChainIsSane) {
2231 #if USE(JSVALUE64)
2232             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2233                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2234 #else
2235             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2236                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2237                 baseReg, propertyReg));
2238 #endif
2239         } else {
2240 #if USE(JSVALUE64)
2241             addSlowPathGenerator(
2242                 slowPathCall(
2243                     outOfBounds, this, operationGetByValStringInt,
2244                     scratchReg, baseReg, propertyReg));
2245 #else
2246             addSlowPathGenerator(
2247                 slowPathCall(
2248                     outOfBounds, this, operationGetByValStringInt,
2249                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2250 #endif
2251         }
2252         
2253 #if USE(JSVALUE64)
2254         jsValueResult(scratchReg, m_currentNode);
2255 #else
2256         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2257 #endif
2258     } else
2259         cellResult(scratchReg, m_currentNode);
2260 }
2261
2262 void SpeculativeJIT::compileFromCharCode(Node* node)
2263 {
2264     Edge& child = node->child1();
2265     if (child.useKind() == UntypedUse) {
2266         JSValueOperand opr(this, child);
2267         JSValueRegs oprRegs = opr.jsValueRegs();
2268
2269         flushRegisters();
2270         JSValueRegsFlushedCallResult result(this);
2271         JSValueRegs resultRegs = result.regs();
2272         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2273         m_jit.exceptionCheck();
2274         
2275         jsValueResult(resultRegs, node);
2276         return;
2277     }
2278
2279     SpeculateStrictInt32Operand property(this, child);
2280     GPRReg propertyReg = property.gpr();
2281     GPRTemporary smallStrings(this);
2282     GPRTemporary scratch(this);
2283     GPRReg scratchReg = scratch.gpr();
2284     GPRReg smallStringsReg = smallStrings.gpr();
2285
2286     JITCompiler::JumpList slowCases;
2287     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2288     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2289     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2290
2291     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2292     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2293     cellResult(scratchReg, m_currentNode);
2294 }
2295
2296 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2297 {
2298     VirtualRegister virtualRegister = node->virtualRegister();
2299     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2300
2301     switch (info.registerFormat()) {
2302     case DataFormatStorage:
2303         RELEASE_ASSERT_NOT_REACHED();
2304
2305     case DataFormatBoolean:
2306     case DataFormatCell:
2307         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2308         return GeneratedOperandTypeUnknown;
2309
2310     case DataFormatNone:
2311     case DataFormatJSCell:
2312     case DataFormatJS:
2313     case DataFormatJSBoolean:
2314     case DataFormatJSDouble:
2315         return GeneratedOperandJSValue;
2316
2317     case DataFormatJSInt32:
2318     case DataFormatInt32:
2319         return GeneratedOperandInteger;
2320
2321     default:
2322         RELEASE_ASSERT_NOT_REACHED();
2323         return GeneratedOperandTypeUnknown;
2324     }
2325 }
2326
2327 void SpeculativeJIT::compileValueToInt32(Node* node)
2328 {
2329     switch (node->child1().useKind()) {
2330 #if USE(JSVALUE64)
2331     case Int52RepUse: {
2332         SpeculateStrictInt52Operand op1(this, node->child1());
2333         GPRTemporary result(this, Reuse, op1);
2334         GPRReg op1GPR = op1.gpr();
2335         GPRReg resultGPR = result.gpr();
2336         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2337         int32Result(resultGPR, node, DataFormatInt32);
2338         return;
2339     }
2340 #endif // USE(JSVALUE64)
2341         
2342     case DoubleRepUse: {
2343         GPRTemporary result(this);
2344         SpeculateDoubleOperand op1(this, node->child1());
2345         FPRReg fpr = op1.fpr();
2346         GPRReg gpr = result.gpr();
2347         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2348         
2349         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2350             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2351         
2352         int32Result(gpr, node);
2353         return;
2354     }
2355     
2356     case NumberUse:
2357     case NotCellUse: {
2358         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2359         case GeneratedOperandInteger: {
2360             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2361             GPRTemporary result(this, Reuse, op1);
2362             m_jit.move(op1.gpr(), result.gpr());
2363             int32Result(result.gpr(), node, op1.format());
2364             return;
2365         }
2366         case GeneratedOperandJSValue: {
2367             GPRTemporary result(this);
2368 #if USE(JSVALUE64)
2369             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2370
2371             GPRReg gpr = op1.gpr();
2372             GPRReg resultGpr = result.gpr();
2373             FPRTemporary tempFpr(this);
2374             FPRReg fpr = tempFpr.fpr();
2375
2376             JITCompiler::Jump isInteger = m_jit.branchIfInt32(gpr);
2377             JITCompiler::JumpList converted;
2378
2379             if (node->child1().useKind() == NumberUse) {
2380                 DFG_TYPE_CHECK(
2381                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2382                     m_jit.branchIfNotNumber(gpr));
2383             } else {
2384                 JITCompiler::Jump isNumber = m_jit.branchIfNumber(gpr);
2385                 
2386                 DFG_TYPE_CHECK(
2387                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2388                 
2389                 // It's not a cell: so true turns into 1 and all else turns into 0.
2390                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2391                 converted.append(m_jit.jump());
2392                 
2393                 isNumber.link(&m_jit);
2394             }
2395
2396             // First, if we get here we have a double encoded as a JSValue
2397             unboxDouble(gpr, resultGpr, fpr);
2398
2399             silentSpillAllRegisters(resultGpr);
2400             callOperation(operationToInt32, resultGpr, fpr);
2401             silentFillAllRegisters();
2402
2403             converted.append(m_jit.jump());
2404
2405             isInteger.link(&m_jit);
2406             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2407
2408             converted.link(&m_jit);
2409 #else
2410             Node* childNode = node->child1().node();
2411             VirtualRegister virtualRegister = childNode->virtualRegister();
2412             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2413
2414             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2415
2416             GPRReg payloadGPR = op1.payloadGPR();
2417             GPRReg resultGpr = result.gpr();
2418         
2419             JITCompiler::JumpList converted;
2420
2421             if (info.registerFormat() == DataFormatJSInt32)
2422                 m_jit.move(payloadGPR, resultGpr);
2423             else {
2424                 GPRReg tagGPR = op1.tagGPR();
2425                 FPRTemporary tempFpr(this);
2426                 FPRReg fpr = tempFpr.fpr();
2427                 FPRTemporary scratch(this);
2428
2429                 JITCompiler::Jump isInteger = m_jit.branchIfInt32(tagGPR);
2430
2431                 if (node->child1().useKind() == NumberUse) {
2432                     DFG_TYPE_CHECK(
2433                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2434                         m_jit.branch32(
2435                             MacroAssembler::AboveOrEqual, tagGPR,
2436                             TrustedImm32(JSValue::LowestTag)));
2437                 } else {
2438                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2439                     
2440                     DFG_TYPE_CHECK(
2441                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2442                         m_jit.branchIfCell(op1.jsValueRegs()));
2443                     
2444                     // It's not a cell: so true turns into 1 and all else turns into 0.
2445                     JITCompiler::Jump isBoolean = m_jit.branchIfBoolean(tagGPR, InvalidGPRReg);
2446                     m_jit.move(TrustedImm32(0), resultGpr);
2447                     converted.append(m_jit.jump());
2448                     
2449                     isBoolean.link(&m_jit);
2450                     m_jit.move(payloadGPR, resultGpr);
2451                     converted.append(m_jit.jump());
2452                     
2453                     isNumber.link(&m_jit);
2454                 }
2455
2456                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2457
2458                 silentSpillAllRegisters(resultGpr);
2459                 callOperation(operationToInt32, resultGpr, fpr);
2460                 silentFillAllRegisters();
2461
2462                 converted.append(m_jit.jump());
2463
2464                 isInteger.link(&m_jit);
2465                 m_jit.move(payloadGPR, resultGpr);
2466
2467                 converted.link(&m_jit);
2468             }
2469 #endif
2470             int32Result(resultGpr, node);
2471             return;
2472         }
2473         case GeneratedOperandTypeUnknown:
2474             RELEASE_ASSERT(!m_compileOkay);
2475             return;
2476         }
2477         RELEASE_ASSERT_NOT_REACHED();
2478         return;
2479     }
2480     
2481     default:
2482         ASSERT(!m_compileOkay);
2483         return;
2484     }
2485 }
2486
2487 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2488 {
2489     if (doesOverflow(node->arithMode())) {
2490         if (enableInt52()) {
2491             SpeculateInt32Operand op1(this, node->child1());
2492             GPRTemporary result(this, Reuse, op1);
2493             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2494             strictInt52Result(result.gpr(), node);
2495             return;
2496         }
2497         SpeculateInt32Operand op1(this, node->child1());
2498         FPRTemporary result(this);
2499             
2500         GPRReg inputGPR = op1.gpr();
2501         FPRReg outputFPR = result.fpr();
2502             
2503         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2504             
2505         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2506         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2507         positive.link(&m_jit);
2508             
2509         doubleResult(outputFPR, node);
2510         return;
2511     }
2512     
2513     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2514
2515     SpeculateInt32Operand op1(this, node->child1());
2516     GPRTemporary result(this);
2517
2518     m_jit.move(op1.gpr(), result.gpr());
2519
2520     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2521
2522     int32Result(result.gpr(), node, op1.format());
2523 }
2524
2525 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2526 {
2527     SpeculateDoubleOperand op1(this, node->child1());
2528     FPRTemporary scratch(this);
2529     GPRTemporary result(this);
2530     
2531     FPRReg valueFPR = op1.fpr();
2532     FPRReg scratchFPR = scratch.fpr();
2533     GPRReg resultGPR = result.gpr();
2534
2535     JITCompiler::JumpList failureCases;
2536     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2537     m_jit.branchConvertDoubleToInt32(
2538         valueFPR, resultGPR, failureCases, scratchFPR,
2539         shouldCheckNegativeZero(node->arithMode()));
2540     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2541
2542     int32Result(resultGPR, node);
2543 }
2544
2545 void SpeculativeJIT::compileDoubleRep(Node* node)
2546 {
2547     switch (node->child1().useKind()) {
2548     case RealNumberUse: {
2549         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2550         FPRTemporary result(this);
2551         
2552         JSValueRegs op1Regs = op1.jsValueRegs();
2553         FPRReg resultFPR = result.fpr();
2554         
2555 #if USE(JSVALUE64)
2556         GPRTemporary temp(this);
2557         GPRReg tempGPR = temp.gpr();
2558         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2559 #else
2560         FPRTemporary temp(this);
2561         FPRReg tempFPR = temp.fpr();
2562         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2563 #endif
2564         
2565         JITCompiler::Jump done = m_jit.branchDouble(
2566             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2567         
2568         DFG_TYPE_CHECK(
2569             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2570         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2571         
2572         done.link(&m_jit);
2573         
2574         doubleResult(resultFPR, node);
2575         return;
2576     }
2577     
2578     case NotCellUse:
2579     case NumberUse: {
2580         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2581
2582         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2583         if (isInt32Speculation(possibleTypes)) {
2584             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2585             FPRTemporary result(this);
2586             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2587             doubleResult(result.fpr(), node);
2588             return;
2589         }
2590
2591         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2592         FPRTemporary result(this);
2593
2594 #if USE(JSVALUE64)
2595         GPRTemporary temp(this);
2596
2597         GPRReg op1GPR = op1.gpr();
2598         GPRReg tempGPR = temp.gpr();
2599         FPRReg resultFPR = result.fpr();
2600         JITCompiler::JumpList done;
2601
2602         JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1GPR);
2603
2604         if (node->child1().useKind() == NotCellUse) {
2605             JITCompiler::Jump isNumber = m_jit.branchIfNumber(op1GPR);
2606             JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1GPR);
2607
2608             static const double zero = 0;
2609             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2610
2611             JITCompiler::Jump isNull = m_jit.branchIfNull(op1GPR);
2612             done.append(isNull);
2613
2614             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2615                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2616
2617             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2618             static const double one = 1;
2619             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2620             done.append(m_jit.jump());
2621             done.append(isFalse);
2622
2623             isUndefined.link(&m_jit);
2624             static const double NaN = PNaN;
2625             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2626             done.append(m_jit.jump());
2627
2628             isNumber.link(&m_jit);
2629         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2630             typeCheck(
2631                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2632                 m_jit.branchIfNotNumber(op1GPR));
2633         }
2634
2635         unboxDouble(op1GPR, tempGPR, resultFPR);
2636         done.append(m_jit.jump());
2637     
2638         isInteger.link(&m_jit);
2639         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2640         done.link(&m_jit);
2641 #else // USE(JSVALUE64) -> this is the 32_64 case
2642         FPRTemporary temp(this);
2643     
2644         GPRReg op1TagGPR = op1.tagGPR();
2645         GPRReg op1PayloadGPR = op1.payloadGPR();
2646         FPRReg tempFPR = temp.fpr();
2647         FPRReg resultFPR = result.fpr();
2648         JITCompiler::JumpList done;
2649     
2650         JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1TagGPR);
2651
2652         if (node->child1().useKind() == NotCellUse) {
2653             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2654             JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1TagGPR);
2655
2656             static const double zero = 0;
2657             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2658
2659             JITCompiler::Jump isNull = m_jit.branchIfNull(op1TagGPR);
2660             done.append(isNull);
2661
2662             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branchIfNotBoolean(op1TagGPR, InvalidGPRReg));
2663
2664             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2665             static const double one = 1;
2666             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2667             done.append(m_jit.jump());
2668             done.append(isFalse);
2669
2670             isUndefined.link(&m_jit);
2671             static const double NaN = PNaN;
2672             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2673             done.append(m_jit.jump());
2674
2675             isNumber.link(&m_jit);
2676         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2677             // This check fails with Int32Tag, but it is OK since Int32 case is already excluded.
2678             typeCheck(
2679                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2680                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2681         }
2682
2683         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2684         done.append(m_jit.jump());
2685     
2686         isInteger.link(&m_jit);
2687         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2688         done.link(&m_jit);
2689 #endif // USE(JSVALUE64)
2690     
2691         doubleResult(resultFPR, node);
2692         return;
2693     }
2694         
2695 #if USE(JSVALUE64)
2696     case Int52RepUse: {
2697         SpeculateStrictInt52Operand value(this, node->child1());
2698         FPRTemporary result(this);
2699         
2700         GPRReg valueGPR = value.gpr();
2701         FPRReg resultFPR = result.fpr();
2702
2703         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2704         
2705         doubleResult(resultFPR, node);
2706         return;
2707     }
2708 #endif // USE(JSVALUE64)
2709         
2710     default:
2711         RELEASE_ASSERT_NOT_REACHED();
2712         return;
2713     }
2714 }
2715
2716 void SpeculativeJIT::compileValueRep(Node* node)
2717 {
2718     switch (node->child1().useKind()) {
2719     case DoubleRepUse: {
2720         SpeculateDoubleOperand value(this, node->child1());
2721         JSValueRegsTemporary result(this);
2722         
2723         FPRReg valueFPR = value.fpr();
2724         JSValueRegs resultRegs = result.regs();
2725         
2726         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2727         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2728         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2729         // local was purified.
2730         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2731             m_jit.purifyNaN(valueFPR);
2732
2733         boxDouble(valueFPR, resultRegs);
2734         
2735         jsValueResult(resultRegs, node);
2736         return;
2737     }
2738         
2739 #if USE(JSVALUE64)
2740     case Int52RepUse: {
2741         SpeculateStrictInt52Operand value(this, node->child1());
2742         GPRTemporary result(this);
2743         
2744         GPRReg valueGPR = value.gpr();
2745         GPRReg resultGPR = result.gpr();
2746         
2747         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2748         
2749         jsValueResult(resultGPR, node);
2750         return;
2751     }
2752 #endif // USE(JSVALUE64)
2753         
2754     default:
2755         RELEASE_ASSERT_NOT_REACHED();
2756         return;
2757     }
2758 }
2759
2760 static double clampDoubleToByte(double d)
2761 {
2762     d += 0.5;
2763     if (!(d > 0))
2764         d = 0;
2765     else if (d > 255)
2766         d = 255;
2767     return d;
2768 }
2769
2770 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2771 {
2772     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2773     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2774     jit.xorPtr(result, result);
2775     MacroAssembler::Jump clamped = jit.jump();
2776     tooBig.link(&jit);
2777     jit.move(JITCompiler::TrustedImm32(255), result);
2778     clamped.link(&jit);
2779     inBounds.link(&jit);
2780 }
2781
2782 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2783 {
2784     // Unordered compare so we pick up NaN
2785     static const double zero = 0;
2786     static const double byteMax = 255;
2787     static const double half = 0.5;
2788     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2789     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2790     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2791     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2792     
2793     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2794     // FIXME: This should probably just use a floating point round!
2795     // https://bugs.webkit.org/show_bug.cgi?id=72054
2796     jit.addDouble(source, scratch);
2797     jit.truncateDoubleToInt32(scratch, result);   
2798     MacroAssembler::Jump truncatedInt = jit.jump();
2799     
2800     tooSmall.link(&jit);
2801     jit.xorPtr(result, result);
2802     MacroAssembler::Jump zeroed = jit.jump();
2803     
2804     tooBig.link(&jit);
2805     jit.move(JITCompiler::TrustedImm32(255), result);
2806     
2807     truncatedInt.link(&jit);
2808     zeroed.link(&jit);
2809
2810 }
2811
2812 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2813 {
2814     if (node->op() == PutByValAlias)
2815         return JITCompiler::Jump();
2816     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2817         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2818     if (view) {
2819         uint32_t length = view->length();
2820         Node* indexNode = m_jit.graph().child(node, 1).node();
2821         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2822             return JITCompiler::Jump();
2823         return m_jit.branch32(
2824             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2825     }
2826     return m_jit.branch32(
2827         MacroAssembler::AboveOrEqual, indexGPR,
2828         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2829 }
2830
2831 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2832 {
2833     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2834     if (!jump.isSet())
2835         return;
2836     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2837 }
2838
2839 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2840 {
2841     JITCompiler::Jump done;
2842     if (outOfBounds.isSet()) {
2843         done = m_jit.jump();
2844         if (node->arrayMode().isInBounds())
2845             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2846         else {
2847             outOfBounds.link(&m_jit);
2848
2849             JITCompiler::Jump notWasteful = m_jit.branch32(
2850                 MacroAssembler::NotEqual,
2851                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2852                 TrustedImm32(WastefulTypedArray));
2853
2854             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2855                 MacroAssembler::Zero,
2856                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2857             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2858             notWasteful.link(&m_jit);
2859         }
2860     }
2861     return done;
2862 }
2863
2864 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2865 {
2866     switch (elementSize(type)) {
2867     case 1:
2868         if (isSigned(type))
2869             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2870         else
2871             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2872         break;
2873     case 2:
2874         if (isSigned(type))
2875             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2876         else
2877             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2878         break;
2879     case 4:
2880         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2881         break;
2882     default:
2883         CRASH();
2884     }
2885 }
2886
2887 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2888 {
2889     if (elementSize(type) < 4 || isSigned(type)) {
2890         int32Result(resultReg, node);
2891         return;
2892     }
2893     
2894     ASSERT(elementSize(type) == 4 && !isSigned(type));
2895     if (node->shouldSpeculateInt32() && canSpeculate) {
2896         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2897         int32Result(resultReg, node);
2898         return;
2899     }
2900     
2901 #if USE(JSVALUE64)
2902     if (node->shouldSpeculateAnyInt()) {
2903         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2904         strictInt52Result(resultReg, node);
2905         return;
2906     }
2907 #endif
2908     
2909     FPRTemporary fresult(this);
2910     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2911     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2912     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2913     positive.link(&m_jit);
2914     doubleResult(fresult.fpr(), node);
2915 }
2916
2917 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2918 {
2919     ASSERT(isInt(type));
2920     
2921     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2922     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2923     StorageOperand storage(this, m_graph.varArgChild(node, 2));
2924
2925     GPRReg baseReg = base.gpr();
2926     GPRReg propertyReg = property.gpr();
2927     GPRReg storageReg = storage.gpr();
2928
2929     GPRTemporary result(this);
2930     GPRReg resultReg = result.gpr();
2931
2932     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2933
2934     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2935     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2936     bool canSpeculate = true;
2937     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2938 }
2939
2940 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2941     GPRTemporary& value,
2942     GPRReg property,
2943 #if USE(JSVALUE32_64)
2944     GPRTemporary& propertyTag,
2945     GPRTemporary& valueTag,
2946 #endif
2947     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2948 {
2949     bool isAppropriateConstant = false;
2950     if (valueUse->isConstant()) {
2951         JSValue jsValue = valueUse->asJSValue();
2952         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2953         SpeculatedType actualType = speculationFromValue(jsValue);
2954         isAppropriateConstant = (expectedType | actualType) == expectedType;
2955     }
2956     
2957     if (isAppropriateConstant) {
2958         JSValue jsValue = valueUse->asJSValue();
2959         if (!jsValue.isNumber()) {
2960             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2961             return false;
2962         }
2963         double d = jsValue.asNumber();
2964         if (isClamped)
2965             d = clampDoubleToByte(d);
2966         GPRTemporary scratch(this);
2967         GPRReg scratchReg = scratch.gpr();
2968         m_jit.move(Imm32(toInt32(d)), scratchReg);
2969         value.adopt(scratch);
2970     } else {
2971         switch (valueUse.useKind()) {
2972         case Int32Use: {
2973             SpeculateInt32Operand valueOp(this, valueUse);
2974             GPRTemporary scratch(this);
2975             GPRReg scratchReg = scratch.gpr();
2976             m_jit.move(valueOp.gpr(), scratchReg);
2977             if (isClamped)
2978                 compileClampIntegerToByte(m_jit, scratchReg);
2979             value.adopt(scratch);
2980             break;
2981         }
2982             
2983 #if USE(JSVALUE64)
2984         case Int52RepUse: {
2985             SpeculateStrictInt52Operand valueOp(this, valueUse);
2986             GPRTemporary scratch(this);
2987             GPRReg scratchReg = scratch.gpr();
2988             m_jit.move(valueOp.gpr(), scratchReg);
2989             if (isClamped) {
2990                 MacroAssembler::Jump inBounds = m_jit.branch64(
2991                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2992                 MacroAssembler::Jump tooBig = m_jit.branch64(
2993                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2994                 m_jit.move(TrustedImm32(0), scratchReg);
2995                 MacroAssembler::Jump clamped = m_jit.jump();
2996                 tooBig.link(&m_jit);
2997                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2998                 clamped.link(&m_jit);
2999                 inBounds.link(&m_jit);
3000             }
3001             value.adopt(scratch);
3002             break;
3003         }
3004 #endif // USE(JSVALUE64)
3005             
3006         case DoubleRepUse: {
3007             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
3008             if (isClamped) {
3009                 SpeculateDoubleOperand valueOp(this, valueUse);
3010                 GPRTemporary result(this);
3011                 FPRTemporary floatScratch(this);
3012                 FPRReg fpr = valueOp.fpr();
3013                 GPRReg gpr = result.gpr();
3014                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
3015                 value.adopt(result);
3016             } else {
3017 #if USE(JSVALUE32_64)
3018                 GPRTemporary realPropertyTag(this);
3019                 propertyTag.adopt(realPropertyTag);
3020                 GPRReg propertyTagGPR = propertyTag.gpr();
3021
3022                 GPRTemporary realValueTag(this);
3023                 valueTag.adopt(realValueTag);
3024                 GPRReg valueTagGPR = valueTag.gpr();
3025 #endif
3026                 SpeculateDoubleOperand valueOp(this, valueUse);
3027                 GPRTemporary result(this);
3028                 FPRReg fpr = valueOp.fpr();
3029                 GPRReg gpr = result.gpr();
3030                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3031                 m_jit.xorPtr(gpr, gpr);
3032                 MacroAssembler::JumpList fixed(m_jit.jump());
3033                 notNaN.link(&m_jit);
3034
3035                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3036                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3037
3038 #if USE(JSVALUE64)
3039                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3040                 boxDouble(fpr, gpr);
3041 #else
3042                 UNUSED_PARAM(property);
3043                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3044                 boxDouble(fpr, valueTagGPR, gpr);
3045 #endif
3046                 slowPathCases.append(m_jit.jump());
3047
3048                 fixed.link(&m_jit);
3049                 value.adopt(result);
3050             }
3051             break;
3052         }
3053             
3054         default:
3055             RELEASE_ASSERT_NOT_REACHED();
3056             break;
3057         }
3058     }
3059     return true;
3060 }
3061
3062 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3063 {
3064     ASSERT(isInt(type));
3065     
3066     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3067     GPRReg storageReg = storage.gpr();
3068     
3069     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3070     
3071     GPRTemporary value;
3072 #if USE(JSVALUE32_64)
3073     GPRTemporary propertyTag;
3074     GPRTemporary valueTag;
3075 #endif
3076
3077     JITCompiler::JumpList slowPathCases;
3078     
3079     bool result = getIntTypedArrayStoreOperand(
3080         value, property,
3081 #if USE(JSVALUE32_64)
3082         propertyTag, valueTag,
3083 #endif
3084         valueUse, slowPathCases, isClamped(type));
3085     if (!result) {
3086         noResult(node);
3087         return;
3088     }
3089
3090     GPRReg valueGPR = value.gpr();
3091 #if USE(JSVALUE32_64)
3092     GPRReg propertyTagGPR = propertyTag.gpr();
3093     GPRReg valueTagGPR = valueTag.gpr();
3094 #endif
3095
3096     ASSERT_UNUSED(valueGPR, valueGPR != property);
3097     ASSERT(valueGPR != base);
3098     ASSERT(valueGPR != storageReg);
3099     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3100
3101     switch (elementSize(type)) {
3102     case 1:
3103         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3104         break;
3105     case 2:
3106         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3107         break;
3108     case 4:
3109         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3110         break;
3111     default:
3112         CRASH();
3113     }
3114
3115     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3116     if (done.isSet())
3117         done.link(&m_jit);
3118
3119     if (!slowPathCases.empty()) {
3120 #if USE(JSVALUE64)
3121         if (node->op() == PutByValDirect) {
3122             addSlowPathGenerator(slowPathCall(
3123                 slowPathCases, this,
3124                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3125                 NoResult, base, property, valueGPR));
3126         } else {
3127             addSlowPathGenerator(slowPathCall(
3128                 slowPathCases, this,
3129                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3130                 NoResult, base, property, valueGPR));
3131         }
3132 #else // not USE(JSVALUE64)
3133         if (node->op() == PutByValDirect) {
3134             addSlowPathGenerator(slowPathCall(
3135                 slowPathCases, this,
3136                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3137                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3138         } else {
3139             addSlowPathGenerator(slowPathCall(
3140                 slowPathCases, this,
3141                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3142                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3143         }
3144 #endif
3145     }
3146     
3147     noResult(node);
3148 }
3149
3150 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3151 {
3152     ASSERT(isFloat(type));
3153     
3154     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3155     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3156     StorageOperand storage(this, m_graph.varArgChild(node, 2));
3157
3158     GPRReg baseReg = base.gpr();
3159     GPRReg propertyReg = property.gpr();
3160     GPRReg storageReg = storage.gpr();
3161
3162     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3163
3164     FPRTemporary result(this);
3165     FPRReg resultReg = result.fpr();
3166     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3167     switch (elementSize(type)) {
3168     case 4:
3169         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3170         m_jit.convertFloatToDouble(resultReg, resultReg);
3171         break;
3172     case 8: {
3173         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3174         break;
3175     }
3176     default:
3177         RELEASE_ASSERT_NOT_REACHED();
3178     }
3179     
3180     doubleResult(resultReg, node);
3181 }
3182
3183 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3184 {
3185     ASSERT(isFloat(type));
3186     
3187     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3188     GPRReg storageReg = storage.gpr();
3189     
3190     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3191     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3192
3193     SpeculateDoubleOperand valueOp(this, valueUse);
3194     FPRTemporary scratch(this);
3195     FPRReg valueFPR = valueOp.fpr();
3196     FPRReg scratchFPR = scratch.fpr();
3197
3198     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3199     
3200     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3201     
3202     switch (elementSize(type)) {
3203     case 4: {
3204         m_jit.moveDouble(valueFPR, scratchFPR);
3205         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3206         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3207         break;
3208     }
3209     case 8:
3210         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3211         break;
3212     default:
3213         RELEASE_ASSERT_NOT_REACHED();
3214     }
3215
3216     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3217     if (done.isSet())
3218         done.link(&m_jit);
3219     noResult(node);
3220 }
3221
3222 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3223 {
3224     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3225     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3226
3227     GPRReg arg1GPR = arg1.gpr();
3228     GPRReg arg2GPR = arg2.gpr();
3229
3230     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3231     speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3232
3233     flushRegisters();
3234     JSValueRegsFlushedCallResult result(this);
3235     JSValueRegs resultRegs = result.regs();
3236     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3237     m_jit.exceptionCheck();
3238
3239     jsValueResult(resultRegs, node);
3240 }
3241
3242 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3243 {
3244     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3245     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3246
3247     GPRReg arg1GPR = arg1.gpr();
3248     GPRReg arg2GPR = arg2.gpr();
3249
3250     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3251     speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3252
3253     flushRegisters();
3254     JSValueRegsFlushedCallResult result(this);
3255     JSValueRegs resultRegs = result.regs();
3256     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3257     m_jit.exceptionCheck();
3258
3259     jsValueResult(resultRegs, node);
3260 }
3261
3262 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3263 {
3264     SpeculateCellOperand arg1(this, child1);
3265     SpeculateCellOperand arg2(this, child2);
3266     JSValueOperand arg3(this, child3);
3267
3268     GPRReg arg1GPR = arg1.gpr();
3269     GPRReg arg2GPR = arg2.gpr();
3270     JSValueRegs arg3Regs = arg3.jsValueRegs();
3271
3272     speculateString(child2, arg2GPR);
3273
3274     flushRegisters();
3275     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3276     m_jit.exceptionCheck();
3277
3278     noResult(node);
3279 }
3280
3281 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3282 {
3283     SpeculateCellOperand arg1(this, child1);
3284     SpeculateCellOperand arg2(this, child2);
3285     JSValueOperand arg3(this, child3);
3286
3287     GPRReg arg1GPR = arg1.gpr();
3288     GPRReg arg2GPR = arg2.gpr();
3289     JSValueRegs arg3Regs = arg3.jsValueRegs();
3290
3291     speculateSymbol(child2, arg2GPR);
3292
3293     flushRegisters();
3294     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3295     m_jit.exceptionCheck();
3296
3297     noResult(node);
3298 }
3299
3300 void SpeculativeJIT::compileGetByValWithThis(Node* node)
3301 {
3302     JSValueOperand base(this, node->child1());
3303     JSValueRegs baseRegs = base.jsValueRegs();
3304     JSValueOperand thisValue(this, node->child2());
3305     JSValueRegs thisValueRegs = thisValue.jsValueRegs();
3306     JSValueOperand subscript(this, node->child3());
3307     JSValueRegs subscriptRegs = subscript.jsValueRegs();
3308
3309     flushRegisters();
3310     JSValueRegsFlushedCallResult result(this);
3311     JSValueRegs resultRegs = result.regs();
3312     callOperation(operationGetByValWithThis, resultRegs, baseRegs, thisValueRegs, subscriptRegs);
3313     m_jit.exceptionCheck();
3314
3315     jsValueResult(resultRegs, node);
3316 }
3317
3318 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3319 {
3320     SpeculateCellOperand base(this, node->child1());
3321
3322     GPRReg baseGPR = base.gpr();
3323
3324     // FIXME: This only works for checking if a single bit is set. If we want to check more
3325     // than one bit at once, we'll need to fix this:
3326     // https://bugs.webkit.org/show_bug.cgi?id=185705
3327     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3328
3329     noResult(node);
3330 }
3331
3332 void SpeculativeJIT::compileParseInt(Node* node)
3333 {
3334     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3335     if (node->child2()) {
3336         SpeculateInt32Operand radix(this, node->child2());
3337         GPRReg radixGPR = radix.gpr();
3338         if (node->child1().useKind() == UntypedUse) {
3339             JSValueOperand value(this, node->child1());
3340             JSValueRegs valueRegs = value.jsValueRegs();
3341
3342             flushRegisters();
3343             JSValueRegsFlushedCallResult result(this);
3344             JSValueRegs resultRegs = result.regs();
3345             callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3346             m_jit.exceptionCheck();
3347             jsValueResult(resultRegs, node);
3348             return;
3349         }
3350
3351         SpeculateCellOperand value(this, node->child1());
3352         GPRReg valueGPR = value.gpr();
3353         speculateString(node->child1(), valueGPR);
3354
3355         flushRegisters();
3356         JSValueRegsFlushedCallResult result(this);
3357         JSValueRegs resultRegs = result.regs();
3358         callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3359         m_jit.exceptionCheck();
3360         jsValueResult(resultRegs, node);
3361         return;
3362     }
3363
3364     if (node->child1().useKind() == UntypedUse) {
3365         JSValueOperand value(this, node->child1());
3366         JSValueRegs valueRegs = value.jsValueRegs();
3367
3368         flushRegisters();
3369         JSValueRegsFlushedCallResult result(this);
3370         JSValueRegs resultRegs = result.regs();
3371         callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3372         m_jit.exceptionCheck();
3373         jsValueResult(resultRegs, node);
3374         return;
3375     }
3376
3377     SpeculateCellOperand value(this, node->child1());
3378     GPRReg valueGPR = value.gpr();
3379     speculateString(node->child1(), valueGPR);
3380
3381     flushRegisters();
3382     JSValueRegsFlushedCallResult result(this);
3383     JSValueRegs resultRegs = result.regs();
3384     callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3385     m_jit.exceptionCheck();
3386     jsValueResult(resultRegs, node);
3387 }
3388
3389 void SpeculativeJIT::compileOverridesHasInstance(Node* node)
3390 {
3391     Node* hasInstanceValueNode = node->child2().node();
3392     JSFunction* defaultHasInstanceFunction = jsCast<JSFunction*>(node->cellOperand()->value());
3393
3394     MacroAssembler::JumpList notDefault;
3395     SpeculateCellOperand base(this, node->child1());
3396     JSValueOperand hasInstanceValue(this, node->child2());
3397     GPRTemporary result(this);
3398
3399     GPRReg baseGPR = base.gpr();
3400     GPRReg resultGPR = result.gpr();
3401
3402     // It would be great if constant folding handled automatically the case where we knew the hasInstance function
3403     // was a constant. Unfortunately, the folding rule for OverridesHasInstance is in the strength reduction phase
3404     // since it relies on OSR information. https://bugs.webkit.org/show_bug.cgi?id=154832
3405     if (!hasInstanceValueNode->isCellConstant() || defaultHasInstanceFunction != hasInstanceValueNode->asCell()) {
3406         JSValueRegs hasInstanceValueRegs = hasInstanceValue.jsValueRegs();
3407 #if USE(JSVALUE64)
3408         notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.gpr(), TrustedImmPtr(node->cellOperand())));
3409 #else
3410         notDefault.append(m_jit.branchIfNotCell(hasInstanceValueRegs));
3411         notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.payloadGPR(), TrustedImmPtr(node->cellOperand())));
3412 #endif
3413     }
3414
3415     // Check that base 'ImplementsDefaultHasInstance'.
3416     m_jit.test8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance), resultGPR);
3417     MacroAssembler::Jump done = m_jit.jump();
3418
3419     if (!notDefault.empty()) {
3420         notDefault.link(&m_jit);
3421         m_jit.move(TrustedImm32(1), resultGPR);
3422     }
3423
3424     done.link(&m_jit);
3425     unblessedBooleanResult(resultGPR, node);
3426 }
3427
3428 void SpeculativeJIT::compileInstanceOfForCells(Node* node, JSValueRegs valueRegs, JSValueRegs prototypeRegs, GPRReg resultGPR, GPRReg scratchGPR, GPRReg scratch2GPR, JITCompiler::Jump slowCase)
3429 {
3430     CallSiteIndex callSiteIndex = m_jit.addCallSite(node->origin.semantic);
3431     
3432     JITInstanceOfGenerator gen(
3433         m_jit.codeBlock(), node->origin.semantic, callSiteIndex, usedRegisters(), resultGPR,
3434         valueRegs.payloadGPR(), prototypeRegs.payloadGPR(), scratchGPR, scratch2GPR,
3435         m_state.forNode(node->child2()).isType(SpecObject | ~SpecCell));
3436     gen.generateFastPath(m_jit);
3437     
3438     JITCompiler::JumpList slowCases;
3439     slowCases.append(slowCase);
3440     
3441     std::unique_ptr<SlowPathGenerator> slowPath = slowPathCall(
3442         slowCases, this, operationInstanceOfOptimize, resultGPR, gen.stubInfo(), valueRegs,
3443         prototypeRegs);
3444     
3445     m_jit.addInstanceOf(gen, slowPath.get());
3446     addSlowPathGenerator(WTFMove(slowPath));
3447 }
3448
3449 void SpeculativeJIT::compileInstanceOf(Node* node)
3450 {
3451 #if USE(JSVALUE64)
3452     if (node->child1().useKind() == CellUse
3453         && node->child2().useKind() == CellUse) {
3454         SpeculateCellOperand value(this, node->child1());
3455         SpeculateCellOperand prototype(this, node->child2());
3456         
3457         GPRTemporary result(this);
3458         GPRTemporary scratch(this);
3459         GPRTemporary scratch2(this);
3460         
3461         GPRReg valueGPR = value.gpr();
3462         GPRReg prototypeGPR = prototype.gpr();
3463         GPRReg resultGPR = result.gpr();
3464         GPRReg scratchGPR = scratch.gpr();
3465         GPRReg scratch2GPR = scratch2.gpr();
3466         
3467         compileInstanceOfForCells(node, JSValueRegs(valueGPR), JSValueRegs(prototypeGPR), resultGPR, scratchGPR, scratch2GPR);
3468         
3469         blessedBooleanResult(resultGPR, node);
3470         return;
3471     }
3472 #endif
3473     
3474     DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse);
3475     DFG_ASSERT(m_jit.graph(), node, node->child2().useKind() == UntypedUse);
3476     
3477     JSValueOperand value(this, node->child1());
3478     JSValueOperand prototype(this, node->child2());
3479     
3480     GPRTemporary result(this);
3481     GPRTemporary scratch(this);
3482     
3483     JSValueRegs valueRegs = value.jsValueRegs();
3484     JSValueRegs prototypeRegs = prototype.jsValueRegs();
3485     
3486     GPRReg resultGPR = result.gpr();
3487     GPRReg scratchGPR = scratch.gpr();
3488     
3489     JITCompiler::Jump isCell = m_jit.branchIfCell(valueRegs);
3490     moveFalseTo(resultGPR);
3491     
3492     JITCompiler::Jump done = m_jit.jump();
3493     
3494     isCell.link(&m_jit);
3495     
3496     JITCompiler::Jump slowCase = m_jit.branchIfNotCell(prototypeRegs);
3497     
3498     compileInstanceOfForCells(node, valueRegs, prototypeRegs, resultGPR, scratchGPR, InvalidGPRReg, slowCase);
3499     
3500     done.link(&m_jit);
3501     blessedBooleanResult(resultGPR, node);
3502     return;
3503 }
3504
3505 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3506 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3507 {
3508     Edge& leftChild = node->child1();
3509     Edge& rightChild = node->child2();
3510
3511     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3512         JSValueOperand left(this, leftChild);
3513         JSValueOperand right(this, rightChild);
3514         JSValueRegs leftRegs = left.jsValueRegs();
3515         JSValueRegs rightRegs = right.jsValueRegs();
3516
3517         flushRegisters();
3518         JSValueRegsFlushedCallResult result(this);
3519         JSValueRegs resultRegs = result.regs();
3520         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3521         m_jit.exceptionCheck();
3522
3523         jsValueResult(resultRegs, node);
3524         return;
3525     }
3526
3527     std::optional<JSValueOperand> left;
3528     std::optional<JSValueOperand> right;
3529
3530     JSValueRegs leftRegs;
3531     JSValueRegs rightRegs;
3532
3533 #if USE(JSVALUE64)
3534     GPRTemporary result(this);
3535     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3536     GPRTemporary scratch(this);
3537     GPRReg scratchGPR = scratch.gpr();
3538 #else
3539     GPRTemporary resultTag(this);
3540     GPRTemporary resultPayload(this);
3541     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3542     GPRReg scratchGPR = resultTag.gpr();
3543 #endif
3544
3545     SnippetOperand leftOperand;
3546     SnippetOperand rightOperand;
3547
3548     // The snippet generator does not support both operands being constant. If the left
3549     // operand is already const, we'll ignore the right operand's constness.
3550     if (leftChild->isInt32Constant())
3551         leftOperand.setConstInt32(leftChild->asInt32());
3552     else if (rightChild->isInt32Constant())
3553         rightOperand.setConstInt32(rightChild->asInt32());
3554
3555     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3556
3557     if (!leftOperand.isConst()) {
3558         left.emplace(this, leftChild);
3559         leftRegs = left->jsValueRegs();
3560     }
3561     if (!rightOperand.isConst()) {
3562         right.emplace(this, rightChild);
3563         rightRegs = right->jsValueRegs();
3564     }
3565
3566     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3567     gen.generateFastPath(m_jit);
3568
3569     ASSERT(gen.didEmitFastPath());
3570     gen.endJumpList().append(m_jit.jump());
3571
3572     gen.slowPathJumpList().link(&m_jit);
3573     silentSpillAllRegisters(resultRegs);
3574
3575     if (leftOperand.isConst()) {
3576         leftRegs = resultRegs;
3577         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3578     } else if (rightOperand.isConst()) {
3579         rightRegs = resultRegs;
3580         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3581     }
3582
3583     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3584
3585     silentFillAllRegisters();
3586     m_jit.exceptionCheck();
3587
3588     gen.endJumpList().link(&m_jit);
3589     jsValueResult(resultRegs, node);
3590 }
3591
3592 void SpeculativeJIT::compileBitwiseOp(Node* node)
3593 {
3594     NodeType op = node->op();
3595     Edge& leftChild = node->child1();
3596     Edge& rightChild = node->child2();
3597
3598     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3599         switch (op) {
3600         case BitAnd:
3601             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3602             return;
3603         case BitOr:
3604             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3605             return;
3606         case BitXor:
3607             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3608             return;
3609         default:
3610             RELEASE_ASSERT_NOT_REACHED();
3611         }
3612     }
3613
3614     if (leftChild->isInt32Constant()) {
3615         SpeculateInt32Operand op2(this, rightChild);
3616         GPRTemporary result(this, Reuse, op2);
3617
3618         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3619
3620         int32Result(result.gpr(), node);
3621
3622     } else if (rightChild->isInt32Constant()) {
3623         SpeculateInt32Operand op1(this, leftChild);
3624         GPRTemporary result(this, Reuse, op1);
3625
3626         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3627
3628         int32Result(result.gpr(), node);
3629
3630     } else {
3631         SpeculateInt32Operand op1(this, leftChild);
3632         SpeculateInt32Operand op2(this, rightChild);
3633         GPRTemporary result(this, Reuse, op1, op2);
3634         
3635         GPRReg reg1 = op1.gpr();
3636         GPRReg reg2 = op2.gpr();
3637         bitOp(op, reg1, reg2, result.gpr());
3638         
3639         int32Result(result.gpr(), node);
3640     }
3641 }
3642
3643 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3644 {
3645     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3646         ? operationValueBitRShift : operationValueBitURShift;
3647     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3648         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3649
3650     Edge& leftChild = node->child1();
3651     Edge& rightChild = node->child2();
3652
3653     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3654         JSValueOperand left(this, leftChild);
3655         JSValueOperand right(this, rightChild);
3656         JSValueRegs leftRegs = left.jsValueRegs();
3657         JSValueRegs rightRegs = right.jsValueRegs();
3658
3659         flushRegisters();
3660         JSValueRegsFlushedCallResult result(this);
3661         JSValueRegs resultRegs = result.regs();
3662         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3663         m_jit.exceptionCheck();
3664
3665         jsValueResult(resultRegs, node);
3666         return;
3667     }
3668
3669     std::optional<JSValueOperand> left;
3670     std::optional<JSValueOperand> right;
3671
3672     JSValueRegs leftRegs;
3673     JSValueRegs rightRegs;
3674
3675     FPRTemporary leftNumber(this);
3676     FPRReg leftFPR = leftNumber.fpr();
3677
3678 #if USE(JSVALUE64)
3679     GPRTemporary result(this);
3680     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3681     GPRTemporary scratch(this);
3682     GPRReg scratchGPR = scratch.gpr();
3683     FPRReg scratchFPR = InvalidFPRReg;
3684 #else
3685     GPRTemporary resultTag(this);
3686     GPRTemporary resultPayload(this);
3687     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3688     GPRReg scratchGPR = resultTag.gpr();
3689     FPRTemporary fprScratch(this);
3690     FPRReg scratchFPR = fprScratch.fpr();
3691 #endif
3692
3693     SnippetOperand leftOperand;
3694     SnippetOperand rightOperand;
3695
3696     // The snippet generator does not support both operands being constant. If the left
3697     // operand is already const, we'll ignore the right operand's constness.
3698     if (leftChild->isInt32Constant())
3699         leftOperand.setConstInt32(leftChild->asInt32());
3700     else if (rightChild->isInt32Constant())
3701         rightOperand.setConstInt32(rightChild->asInt32());
3702
3703     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3704
3705     if (!leftOperand.isConst()) {
3706         left.emplace(this, leftChild);
3707         leftRegs = left->jsValueRegs();
3708     }
3709     if (!rightOperand.isConst()) {
3710         right.emplace(this, rightChild);
3711         rightRegs = right->jsValueRegs();
3712     }
3713
3714     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3715         leftFPR, scratchGPR, scratchFPR, shiftType);
3716     gen.generateFastPath(m_jit);
3717
3718     ASSERT(gen.didEmitFastPath());
3719     gen.endJumpList().append(m_jit.jump());
3720
3721     gen.slowPathJumpList().link(&m_jit);
3722     silentSpillAllRegisters(resultRegs);
3723
3724     if (leftOperand.isConst()) {
3725         leftRegs = resultRegs;
3726         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3727     } else if (rightOperand.isConst()) {
3728         rightRegs = resultRegs;
3729         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3730     }
3731
3732     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3733
3734     silentFillAllRegisters();
3735     m_jit.exceptionCheck();
3736
3737     gen.endJumpList().link(&m_jit);
3738     jsValueResult(resultRegs, node);
3739     return;
3740 }
3741
3742 void SpeculativeJIT::compileShiftOp(Node* node)
3743 {
3744     NodeType op = node->op();
3745     Edge& leftChild = node->child1();
3746     Edge& rightChild = node->child2();
3747
3748     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3749         switch (op) {
3750         case BitLShift:
3751             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3752             return;
3753         case BitRShift:
3754         case BitURShift:
3755             emitUntypedRightShiftBitOp(node);
3756             return;
3757         default:
3758             RELEASE_ASSERT_NOT_REACHED();
3759         }
3760     }
3761
3762     if (rightChild->isInt32Constant()) {
3763         SpeculateInt32Operand op1(this, leftChild);
3764         GPRTemporary result(this, Reuse, op1);
3765
3766         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3767
3768         int32Result(result.gpr(), node);
3769     } else {
3770         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3771         SpeculateInt32Operand op1(this, leftChild);
3772         SpeculateInt32Operand op2(this, rightChild);
3773         GPRTemporary result(this, Reuse, op1);
3774
3775         GPRReg reg1 = op1.gpr();
3776         GPRReg reg2 = op2.gpr();
3777         shiftOp(op, reg1, reg2, result.gpr());
3778
3779         int32Result(result.gpr(), node);
3780     }
3781 }
3782
3783 void SpeculativeJIT::compileValueAdd(Node* node)
3784 {
3785     Edge& leftChild = node->child1();
3786     Edge& rightChild = node->child2();
3787
3788     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3789         JSValueOperand left(this, leftChild);
3790         JSValueOperand right(this, rightChild);
3791         JSValueRegs leftRegs = left.jsValueRegs();
3792         JSValueRegs rightRegs = right.jsValueRegs();
3793
3794         flushRegisters();
3795         JSValueRegsFlushedCallResult result(this);
3796         JSValueRegs resultRegs = result.regs();
3797         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3798         m_jit.exceptionCheck();
3799     
3800         jsValueResult(resultRegs, node);
3801         return;
3802     }
3803
3804 #if USE(JSVALUE64)
3805     bool needsScratchGPRReg = true;
3806     bool needsScratchFPRReg = false;
3807 #else
3808     bool needsScratchGPRReg = true;
3809     bool needsScratchFPRReg = true;
3810 #endif
3811
3812     CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3813     ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(node->origin.semantic.bytecodeIndex);
3814     Instruction* instruction = &baselineCodeBlock->instructions()[node->origin.semantic.bytecodeIndex];
3815     JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile, instruction);
3816     auto repatchingFunction = operationValueAddOptimize;
3817     auto nonRepatchingFunction = operationValueAdd;
3818     
3819     compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3820 }
3821
3822 template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3823 void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3824 {
3825     Edge& leftChild = node->child1();
3826     Edge& rightChild = node->child2();
3827
3828     std::optional<JSValueOperand> left;
3829     std::optional<JSValueOperand> right;
3830
3831     JSValueRegs leftRegs;
3832     JSValueRegs rightRegs;
3833
3834     FPRTemporary leftNumber(this);
3835     FPRTemporary rightNumber(this);
3836     FPRReg leftFPR = leftNumber.fpr();
3837     FPRReg rightFPR = rightNumber.fpr();
3838
3839     GPRReg scratchGPR = InvalidGPRReg;
3840     FPRReg scratchFPR = InvalidFPRReg;
3841
3842     std::optional<FPRTemporary> fprScratch;
3843     if (needsScratchFPRReg) {
3844         fprScratch.emplace(this);
3845         scratchFPR = fprScratch->fpr();
3846     }
3847
3848 #if USE(JSVALUE64)
3849     std::optional<GPRTemporary> gprScratch;
3850     if (needsScratchGPRReg) {
3851         gprScratch.emplace(this);
3852         scratchGPR = gprScratch->gpr();
3853     }
3854     GPRTemporary result(this);
3855     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3856 #else
3857     GPRTemporary resultTag(this);
3858     GPRTemporary resultPayload(this);
3859     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3860     if (needsScratchGPRReg)
3861         scratchGPR = resultRegs.tagGPR();
3862 #endif
3863
3864     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3865     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3866
3867     // The snippet generator does not support both operands being constant. If the left
3868     // operand is already const, we'll ignore the right operand's constness.
3869     if (leftChild->isInt32Constant())
3870         leftOperand.setConstInt32(leftChild->asInt32());
3871     else if (rightChild->isInt32Constant())
3872         rightOperand.setConstInt32(rightChild->asInt32());
3873
3874     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3875     ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
3876
3877     if (!Generator::isLeftOperandValidConstant(leftOperand)) {
3878         left.emplace(this, leftChild);
3879         leftRegs = left->jsValueRegs();
3880     }
3881     if (!Generator::isRightOperandValidConstant(rightOperand)) {
3882         right.emplace(this, rightChild);
3883         rightRegs = right->jsValueRegs();
3884     }
3885
3886 #if ENABLE(MATH_IC_STATS)
3887     auto inlineStart = m_jit.label();
3888 #endif
3889
3890     Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
3891     mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
3892
3893     bool shouldEmitProfiling = false;
3894     bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
3895     if (generatedInline) {
3896         ASSERT(!addICGenerationState->slowPathJumps.empty());
3897
3898         Vector<SilentRegisterSavePlan> savePlans;
3899         silentSpillAllRegistersImpl(false, savePlans, resultRegs);
3900
3901         auto done = m_jit.label();
3902
3903         addSlowPathGenerator([=, savePlans = WTFMove(savePlans)] () {
3904             addICGenerationState->slowPathJumps.link(&m_jit);
3905             addICGenerationState->slowPathStart = m_jit.label();
3906 #if ENABLE(MATH_IC_STATS)
3907             auto slowPathStart = m_jit.label();
3908 #endif
3909
3910             silentSpill(savePlans);
3911
3912             auto innerLeftRegs = leftRegs;
3913             auto innerRightRegs = rightRegs;
3914             if (Generator::isLeftOperandValidConstant(leftOperand)) {
3915                 innerLeftRegs = resultRegs;
3916                 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
3917             } else if (Generator::isRightOperandValidConstant(rightOperand)) {
3918                 innerRightRegs = resultRegs;
3919                 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
3920             }
3921
3922             if (addICGenerationState->shouldSlowPathRepatch)
3923                 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
3924             else
3925             &nbs