JSObject shouldn't do index masking
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCallCreateDirectArgumentsWithKnownLengthSlowPathGenerator.h"
37 #include "DFGCapabilities.h"
38 #include "DFGMayExit.h"
39 #include "DFGOSRExitFuzz.h"
40 #include "DFGSaneStringGetByValSlowPathGenerator.h"
41 #include "DFGSlowPathGenerator.h"
42 #include "DFGSnippetParams.h"
43 #include "DirectArguments.h"
44 #include "JITAddGenerator.h"
45 #include "JITBitAndGenerator.h"
46 #include "JITBitOrGenerator.h"
47 #include "JITBitXorGenerator.h"
48 #include "JITDivGenerator.h"
49 #include "JITLeftShiftGenerator.h"
50 #include "JITMulGenerator.h"
51 #include "JITRightShiftGenerator.h"
52 #include "JITSubGenerator.h"
53 #include "JSAsyncFunction.h"
54 #include "JSAsyncGeneratorFunction.h"
55 #include "JSCInlines.h"
56 #include "JSFixedArray.h"
57 #include "JSGeneratorFunction.h"
58 #include "JSLexicalEnvironment.h"
59 #include "JSPropertyNameEnumerator.h"
60 #include "LinkBuffer.h"
61 #include "RegExpConstructor.h"
62 #include "ScopedArguments.h"
63 #include "ScratchRegisterAllocator.h"
64 #include "SuperSampler.h"
65 #include "WeakMapImpl.h"
66 #include <wtf/BitVector.h>
67 #include <wtf/Box.h>
68 #include <wtf/MathExtras.h>
69
70 namespace JSC { namespace DFG {
71
72 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
73     : m_compileOkay(true)
74     , m_jit(jit)
75     , m_graph(m_jit.graph())
76     , m_currentNode(0)
77     , m_lastGeneratedNode(LastNodeType)
78     , m_indexInBlock(0)
79     , m_generationInfo(m_jit.graph().frameRegisterCount())
80     , m_state(m_jit.graph())
81     , m_interpreter(m_jit.graph(), m_state)
82     , m_stream(&jit.jitCode()->variableEventStream)
83     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
84 {
85 }
86
87 SpeculativeJIT::~SpeculativeJIT()
88 {
89 }
90
91 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
92 {
93     IndexingType indexingType = structure->indexingType();
94     bool hasIndexingHeader = hasIndexedProperties(indexingType);
95
96     unsigned inlineCapacity = structure->inlineCapacity();
97     unsigned outOfLineCapacity = structure->outOfLineCapacity();
98     
99     GPRTemporary scratch(this);
100     GPRTemporary scratch2(this);
101     GPRReg scratchGPR = scratch.gpr();
102     GPRReg scratch2GPR = scratch2.gpr();
103
104     ASSERT(vectorLength >= numElements);
105     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
106     
107     JITCompiler::JumpList slowCases;
108
109     size_t size = 0;
110     if (hasIndexingHeader)
111         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
112     size += outOfLineCapacity * sizeof(JSValue);
113
114     m_jit.move(TrustedImmPtr(nullptr), storageGPR);
115
116     if (size) {
117         if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
118             m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
119             
120             m_jit.addPtr(
121                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
122                 storageGPR);
123             
124             if (hasIndexingHeader)
125                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
126         } else
127             slowCases.append(m_jit.jump());
128     }
129
130     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
131     Allocator allocator = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
132     if (allocator) {
133         emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
134         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
135     } else
136         slowCases.append(m_jit.jump());
137
138     // I want a slow path that also loads out the storage pointer, and that's
139     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
140     // of work for a very small piece of functionality. :-/
141     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
142         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
143         structure, vectorLength));
144
145     if (numElements < vectorLength) {
146 #if USE(JSVALUE64)
147         if (hasDouble(structure->indexingType()))
148             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
149         else
150             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
151         for (unsigned i = numElements; i < vectorLength; ++i)
152             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
153 #else
154         EncodedValueDescriptor value;
155         if (hasDouble(structure->indexingType()))
156             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
157         else
158             value.asInt64 = JSValue::encode(JSValue());
159         for (unsigned i = numElements; i < vectorLength; ++i) {
160             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
161             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
162         }
163 #endif
164     }
165     
166     if (hasIndexingHeader)
167         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
168     
169     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
170     
171     m_jit.mutatorFence(*m_jit.vm());
172 }
173
174 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
175 {
176     if (inlineCallFrame && !inlineCallFrame->isVarargs())
177         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
178     else {
179         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
180         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
181         if (!includeThis)
182             m_jit.sub32(TrustedImm32(1), lengthGPR);
183     }
184 }
185
186 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
187 {
188     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
189 }
190
191 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
192 {
193     if (origin.inlineCallFrame) {
194         if (origin.inlineCallFrame->isClosureCall) {
195             m_jit.loadPtr(
196                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
197                 calleeGPR);
198         } else {
199             m_jit.move(
200                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
201                 calleeGPR);
202         }
203     } else
204         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
205 }
206
207 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
208 {
209     m_jit.addPtr(
210         TrustedImm32(
211             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
212         GPRInfo::callFrameRegister, startGPR);
213 }
214
215 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
216 {
217     if (!Options::useOSRExitFuzz()
218         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
219         || !doOSRExitFuzzing())
220         return MacroAssembler::Jump();
221     
222     MacroAssembler::Jump result;
223     
224     m_jit.pushToSave(GPRInfo::regT0);
225     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
226     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
227     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
228     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
229     unsigned at = Options::fireOSRExitFuzzAt();
230     if (at || atOrAfter) {
231         unsigned threshold;
232         MacroAssembler::RelationalCondition condition;
233         if (atOrAfter) {
234             threshold = atOrAfter;
235             condition = MacroAssembler::Below;
236         } else {
237             threshold = at;
238             condition = MacroAssembler::NotEqual;
239         }
240         MacroAssembler::Jump ok = m_jit.branch32(
241             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
242         m_jit.popToRestore(GPRInfo::regT0);
243         result = m_jit.jump();
244         ok.link(&m_jit);
245     }
246     m_jit.popToRestore(GPRInfo::regT0);
247     
248     return result;
249 }
250
251 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
252 {
253     if (!m_compileOkay)
254         return;
255     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
256     if (fuzzJump.isSet()) {
257         JITCompiler::JumpList jumpsToFail;
258         jumpsToFail.append(fuzzJump);
259         jumpsToFail.append(jumpToFail);
260         m_jit.appendExitInfo(jumpsToFail);
261     } else
262         m_jit.appendExitInfo(jumpToFail);
263     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
264 }
265
266 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
267 {
268     if (!m_compileOkay)
269         return;
270     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
271     if (fuzzJump.isSet()) {
272         JITCompiler::JumpList myJumpsToFail;
273         myJumpsToFail.append(jumpsToFail);
274         myJumpsToFail.append(fuzzJump);
275         m_jit.appendExitInfo(myJumpsToFail);
276     } else
277         m_jit.appendExitInfo(jumpsToFail);
278     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
279 }
280
281 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
282 {
283     if (!m_compileOkay)
284         return OSRExitJumpPlaceholder();
285     unsigned index = m_jit.jitCode()->osrExit.size();
286     m_jit.appendExitInfo();
287     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
288     return OSRExitJumpPlaceholder(index);
289 }
290
291 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
292 {
293     return speculationCheck(kind, jsValueSource, nodeUse.node());
294 }
295
296 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
297 {
298     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
299 }
300
301 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
302 {
303     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
304 }
305
306 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
307 {
308     if (!m_compileOkay)
309         return;
310     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
311     m_jit.appendExitInfo(jumpToFail);
312     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
313 }
314
315 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
316 {
317     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
318 }
319
320 void SpeculativeJIT::emitInvalidationPoint(Node* node)
321 {
322     if (!m_compileOkay)
323         return;
324     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
325     m_jit.jitCode()->appendOSRExit(OSRExit(
326         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
327         this, m_stream->size()));
328     info.m_replacementSource = m_jit.watchpointLabel();
329     ASSERT(info.m_replacementSource.isSet());
330     noResult(node);
331 }
332
333 void SpeculativeJIT::unreachable(Node* node)
334 {
335     m_compileOkay = false;
336     m_jit.abortWithReason(DFGUnreachableNode, node->op());
337 }
338
339 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
340 {
341     if (!m_compileOkay)
342         return;
343     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
344     m_compileOkay = false;
345     if (verboseCompilationEnabled())
346         dataLog("Bailing compilation.\n");
347 }
348
349 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
350 {
351     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
352 }
353
354 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
355 {
356     ASSERT(needsTypeCheck(edge, typesPassedThrough));
357     m_interpreter.filter(edge, typesPassedThrough);
358     speculationCheck(exitKind, source, edge.node(), jumpToFail);
359 }
360
361 RegisterSet SpeculativeJIT::usedRegisters()
362 {
363     RegisterSet result;
364     
365     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
366         GPRReg gpr = GPRInfo::toRegister(i);
367         if (m_gprs.isInUse(gpr))
368             result.set(gpr);
369     }
370     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
371         FPRReg fpr = FPRInfo::toRegister(i);
372         if (m_fprs.isInUse(fpr))
373             result.set(fpr);
374     }
375     
376     result.merge(RegisterSet::stubUnavailableRegisters());
377     
378     return result;
379 }
380
381 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
382 {
383     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
384 }
385
386 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
387 {
388     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
389 }
390
391 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
392 {
393     for (auto& slowPathGenerator : m_slowPathGenerators) {
394         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
395         slowPathGenerator->generate(this);
396     }
397     for (auto& slowPathLambda : m_slowPathLambdas) {
398         Node* currentNode = slowPathLambda.currentNode;
399         m_currentNode = currentNode;
400         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
401         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
402         slowPathLambda.generator();
403         m_outOfLineStreamIndex = std::nullopt;
404     }
405 }
406
407 void SpeculativeJIT::clearGenerationInfo()
408 {
409     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
410         m_generationInfo[i] = GenerationInfo();
411     m_gprs = RegisterBank<GPRInfo>();
412     m_fprs = RegisterBank<FPRInfo>();
413 }
414
415 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
416 {
417     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
418     Node* node = info.node();
419     DataFormat registerFormat = info.registerFormat();
420     ASSERT(registerFormat != DataFormatNone);
421     ASSERT(registerFormat != DataFormatDouble);
422         
423     SilentSpillAction spillAction;
424     SilentFillAction fillAction;
425         
426     if (!info.needsSpill())
427         spillAction = DoNothingForSpill;
428     else {
429 #if USE(JSVALUE64)
430         ASSERT(info.gpr() == source);
431         if (registerFormat == DataFormatInt32)
432             spillAction = Store32Payload;
433         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
434             spillAction = StorePtr;
435         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
436             spillAction = Store64;
437         else {
438             ASSERT(registerFormat & DataFormatJS);
439             spillAction = Store64;
440         }
441 #elif USE(JSVALUE32_64)
442         if (registerFormat & DataFormatJS) {
443             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
444             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
445         } else {
446             ASSERT(info.gpr() == source);
447             spillAction = Store32Payload;
448         }
449 #endif
450     }
451         
452     if (registerFormat == DataFormatInt32) {
453         ASSERT(info.gpr() == source);
454         ASSERT(isJSInt32(info.registerFormat()));
455         if (node->hasConstant()) {
456             ASSERT(node->isInt32Constant());
457             fillAction = SetInt32Constant;
458         } else
459             fillAction = Load32Payload;
460     } else if (registerFormat == DataFormatBoolean) {
461 #if USE(JSVALUE64)
462         RELEASE_ASSERT_NOT_REACHED();
463 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
464         fillAction = DoNothingForFill;
465 #endif
466 #elif USE(JSVALUE32_64)
467         ASSERT(info.gpr() == source);
468         if (node->hasConstant()) {
469             ASSERT(node->isBooleanConstant());
470             fillAction = SetBooleanConstant;
471         } else
472             fillAction = Load32Payload;
473 #endif
474     } else if (registerFormat == DataFormatCell) {
475         ASSERT(info.gpr() == source);
476         if (node->hasConstant()) {
477             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
478             node->asCell(); // To get the assertion.
479             fillAction = SetCellConstant;
480         } else {
481 #if USE(JSVALUE64)
482             fillAction = LoadPtr;
483 #else
484             fillAction = Load32Payload;
485 #endif
486         }
487     } else if (registerFormat == DataFormatStorage) {
488         ASSERT(info.gpr() == source);
489         fillAction = LoadPtr;
490     } else if (registerFormat == DataFormatInt52) {
491         if (node->hasConstant())
492             fillAction = SetInt52Constant;
493         else if (info.spillFormat() == DataFormatInt52)
494             fillAction = Load64;
495         else if (info.spillFormat() == DataFormatStrictInt52)
496             fillAction = Load64ShiftInt52Left;
497         else if (info.spillFormat() == DataFormatNone)
498             fillAction = Load64;
499         else {
500             RELEASE_ASSERT_NOT_REACHED();
501 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
502             fillAction = Load64; // Make GCC happy.
503 #endif
504         }
505     } else if (registerFormat == DataFormatStrictInt52) {
506         if (node->hasConstant())
507             fillAction = SetStrictInt52Constant;
508         else if (info.spillFormat() == DataFormatInt52)
509             fillAction = Load64ShiftInt52Right;
510         else if (info.spillFormat() == DataFormatStrictInt52)
511             fillAction = Load64;
512         else if (info.spillFormat() == DataFormatNone)
513             fillAction = Load64;
514         else {
515             RELEASE_ASSERT_NOT_REACHED();
516 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
517             fillAction = Load64; // Make GCC happy.
518 #endif
519         }
520     } else {
521         ASSERT(registerFormat & DataFormatJS);
522 #if USE(JSVALUE64)
523         ASSERT(info.gpr() == source);
524         if (node->hasConstant()) {
525             if (node->isCellConstant())
526                 fillAction = SetTrustedJSConstant;
527             else
528                 fillAction = SetJSConstant;
529         } else if (info.spillFormat() == DataFormatInt32) {
530             ASSERT(registerFormat == DataFormatJSInt32);
531             fillAction = Load32PayloadBoxInt;
532         } else
533             fillAction = Load64;
534 #else
535         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
536         if (node->hasConstant())
537             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
538         else if (info.payloadGPR() == source)
539             fillAction = Load32Payload;
540         else { // Fill the Tag
541             switch (info.spillFormat()) {
542             case DataFormatInt32:
543                 ASSERT(registerFormat == DataFormatJSInt32);
544                 fillAction = SetInt32Tag;
545                 break;
546             case DataFormatCell:
547                 ASSERT(registerFormat == DataFormatJSCell);
548                 fillAction = SetCellTag;
549                 break;
550             case DataFormatBoolean:
551                 ASSERT(registerFormat == DataFormatJSBoolean);
552                 fillAction = SetBooleanTag;
553                 break;
554             default:
555                 fillAction = Load32Tag;
556                 break;
557             }
558         }
559 #endif
560     }
561         
562     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
563 }
564     
565 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
566 {
567     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
568     Node* node = info.node();
569     ASSERT(info.registerFormat() == DataFormatDouble);
570
571     SilentSpillAction spillAction;
572     SilentFillAction fillAction;
573         
574     if (!info.needsSpill())
575         spillAction = DoNothingForSpill;
576     else {
577         ASSERT(!node->hasConstant());
578         ASSERT(info.spillFormat() == DataFormatNone);
579         ASSERT(info.fpr() == source);
580         spillAction = StoreDouble;
581     }
582         
583 #if USE(JSVALUE64)
584     if (node->hasConstant()) {
585         node->asNumber(); // To get the assertion.
586         fillAction = SetDoubleConstant;
587     } else {
588         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
589         fillAction = LoadDouble;
590     }
591 #elif USE(JSVALUE32_64)
592     ASSERT(info.registerFormat() == DataFormatDouble);
593     if (node->hasConstant()) {
594         node->asNumber(); // To get the assertion.
595         fillAction = SetDoubleConstant;
596     } else
597         fillAction = LoadDouble;
598 #endif
599
600     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
601 }
602     
603 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
604 {
605     switch (plan.spillAction()) {
606     case DoNothingForSpill:
607         break;
608     case Store32Tag:
609         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
610         break;
611     case Store32Payload:
612         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
613         break;
614     case StorePtr:
615         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
616         break;
617 #if USE(JSVALUE64)
618     case Store64:
619         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
620         break;
621 #endif
622     case StoreDouble:
623         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
624         break;
625     default:
626         RELEASE_ASSERT_NOT_REACHED();
627     }
628 }
629     
630 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
631 {
632     switch (plan.fillAction()) {
633     case DoNothingForFill:
634         break;
635     case SetInt32Constant:
636         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
637         break;
638 #if USE(JSVALUE64)
639     case SetInt52Constant:
640         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
641         break;
642     case SetStrictInt52Constant:
643         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
644         break;
645 #endif // USE(JSVALUE64)
646     case SetBooleanConstant:
647         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
648         break;
649     case SetCellConstant:
650         ASSERT(plan.node()->constant()->value().isCell());
651         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
652         break;
653 #if USE(JSVALUE64)
654     case SetTrustedJSConstant:
655         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
656         break;
657     case SetJSConstant:
658         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
659         break;
660     case SetDoubleConstant:
661         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
662         break;
663     case Load32PayloadBoxInt:
664         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
665         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
666         break;
667     case Load32PayloadConvertToInt52:
668         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
669         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
670         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
671         break;
672     case Load32PayloadSignExtend:
673         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
674         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
675         break;
676 #else
677     case SetJSConstantTag:
678         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
679         break;
680     case SetJSConstantPayload:
681         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
682         break;
683     case SetInt32Tag:
684         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
685         break;
686     case SetCellTag:
687         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
688         break;
689     case SetBooleanTag:
690         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
691         break;
692     case SetDoubleConstant:
693         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
694         break;
695 #endif
696     case Load32Tag:
697         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
698         break;
699     case Load32Payload:
700         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
701         break;
702     case LoadPtr:
703         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
704         break;
705 #if USE(JSVALUE64)
706     case Load64:
707         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
708         break;
709     case Load64ShiftInt52Right:
710         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
711         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
712         break;
713     case Load64ShiftInt52Left:
714         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
715         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
716         break;
717 #endif
718     case LoadDouble:
719         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
720         break;
721     default:
722         RELEASE_ASSERT_NOT_REACHED();
723     }
724 }
725
726 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
727 {
728     JITCompiler::JumpList result;
729     
730     switch (arrayMode.type()) {
731     case Array::Int32:
732     case Array::Double:
733     case Array::Contiguous:
734     case Array::Undecided:
735     case Array::ArrayStorage: {
736         IndexingType shape = arrayMode.shapeMask();
737         switch (arrayMode.arrayClass()) {
738         case Array::OriginalArray:
739             RELEASE_ASSERT_NOT_REACHED();
740             return result;
741
742         case Array::Array:
743             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
744             result.append(m_jit.branch32(
745                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
746             return result;
747
748         case Array::NonArray:
749         case Array::OriginalNonArray:
750             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
751             result.append(m_jit.branch32(
752                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
753             return result;
754
755         case Array::PossiblyArray:
756             m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
757             result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
758             return result;
759         }
760
761         RELEASE_ASSERT_NOT_REACHED();
762         return result;
763     }
764
765     case Array::SlowPutArrayStorage: {
766         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
767
768         switch (arrayMode.arrayClass()) {
769         case Array::OriginalArray:
770             RELEASE_ASSERT_NOT_REACHED();
771             return result;
772
773         case Array::Array:
774             result.append(
775                 m_jit.branchTest32(
776                     MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
777             break;
778
779         case Array::NonArray:
780         case Array::OriginalNonArray:
781             result.append(
782                 m_jit.branchTest32(
783                     MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
784             break;
785
786         case Array::PossiblyArray:
787             break;
788         }
789
790         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
791         m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
792         result.append(
793             m_jit.branch32(
794                 MacroAssembler::Above, tempGPR,
795                 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
796         return result;
797     }
798     default:
799         CRASH();
800         break;
801     }
802     
803     return result;
804 }
805
806 void SpeculativeJIT::checkArray(Node* node)
807 {
808     ASSERT(node->arrayMode().isSpecific());
809     ASSERT(!node->arrayMode().doesConversion());
810     
811     SpeculateCellOperand base(this, node->child1());
812     GPRReg baseReg = base.gpr();
813     
814     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
815         noResult(m_currentNode);
816         return;
817     }
818     
819     switch (node->arrayMode().type()) {
820     case Array::AnyTypedArray:
821     case Array::String:
822         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
823         return;
824     case Array::Int32:
825     case Array::Double:
826     case Array::Contiguous:
827     case Array::Undecided:
828     case Array::ArrayStorage:
829     case Array::SlowPutArrayStorage: {
830         GPRTemporary temp(this);
831         GPRReg tempGPR = temp.gpr();
832         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
833         speculationCheck(
834             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
835             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
836         
837         noResult(m_currentNode);
838         return;
839     }
840     case Array::DirectArguments:
841         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
842         noResult(m_currentNode);
843         return;
844     case Array::ScopedArguments:
845         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
846         noResult(m_currentNode);
847         return;
848     default:
849         speculateCellTypeWithoutTypeFiltering(
850             node->child1(), baseReg,
851             typeForTypedArrayType(node->arrayMode().typedArrayType()));
852         noResult(m_currentNode);
853         return;
854     }
855 }
856
857 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
858 {
859     ASSERT(node->arrayMode().doesConversion());
860     
861     GPRTemporary temp(this);
862     GPRTemporary structure;
863     GPRReg tempGPR = temp.gpr();
864     GPRReg structureGPR = InvalidGPRReg;
865     
866     if (node->op() != ArrayifyToStructure) {
867         GPRTemporary realStructure(this);
868         structure.adopt(realStructure);
869         structureGPR = structure.gpr();
870     }
871         
872     // We can skip all that comes next if we already have array storage.
873     MacroAssembler::JumpList slowPath;
874     
875     if (node->op() == ArrayifyToStructure) {
876         slowPath.append(m_jit.branchWeakStructure(
877             JITCompiler::NotEqual,
878             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
879             node->structure()));
880     } else {
881         m_jit.load8(
882             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
883         
884         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
885     }
886     
887     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
888         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
889     
890     noResult(m_currentNode);
891 }
892
893 void SpeculativeJIT::arrayify(Node* node)
894 {
895     ASSERT(node->arrayMode().isSpecific());
896     
897     SpeculateCellOperand base(this, node->child1());
898     
899     if (!node->child2()) {
900         arrayify(node, base.gpr(), InvalidGPRReg);
901         return;
902     }
903     
904     SpeculateInt32Operand property(this, node->child2());
905     
906     arrayify(node, base.gpr(), property.gpr());
907 }
908
909 GPRReg SpeculativeJIT::fillStorage(Edge edge)
910 {
911     VirtualRegister virtualRegister = edge->virtualRegister();
912     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
913     
914     switch (info.registerFormat()) {
915     case DataFormatNone: {
916         if (info.spillFormat() == DataFormatStorage) {
917             GPRReg gpr = allocate();
918             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
919             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
920             info.fillStorage(*m_stream, gpr);
921             return gpr;
922         }
923         
924         // Must be a cell; fill it as a cell and then return the pointer.
925         return fillSpeculateCell(edge);
926     }
927         
928     case DataFormatStorage: {
929         GPRReg gpr = info.gpr();
930         m_gprs.lock(gpr);
931         return gpr;
932     }
933         
934     default:
935         return fillSpeculateCell(edge);
936     }
937 }
938
939 void SpeculativeJIT::useChildren(Node* node)
940 {
941     if (node->flags() & NodeHasVarArgs) {
942         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
943             if (!!m_jit.graph().m_varArgChildren[childIdx])
944                 use(m_jit.graph().m_varArgChildren[childIdx]);
945         }
946     } else {
947         Edge child1 = node->child1();
948         if (!child1) {
949             ASSERT(!node->child2() && !node->child3());
950             return;
951         }
952         use(child1);
953         
954         Edge child2 = node->child2();
955         if (!child2) {
956             ASSERT(!node->child3());
957             return;
958         }
959         use(child2);
960         
961         Edge child3 = node->child3();
962         if (!child3)
963             return;
964         use(child3);
965     }
966 }
967
968 void SpeculativeJIT::compileTryGetById(Node* node)
969 {
970     switch (node->child1().useKind()) {
971     case CellUse: {
972         SpeculateCellOperand base(this, node->child1());
973         JSValueRegsTemporary result(this, Reuse, base);
974
975         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
976         JSValueRegs resultRegs = result.regs();
977
978         base.use();
979
980         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, AccessType::TryGet);
981
982         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
983         break;
984     }
985
986     case UntypedUse: {
987         JSValueOperand base(this, node->child1());
988         JSValueRegsTemporary result(this, Reuse, base);
989
990         JSValueRegs baseRegs = base.jsValueRegs();
991         JSValueRegs resultRegs = result.regs();
992
993         base.use();
994
995         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
996
997         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, AccessType::TryGet);
998
999         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1000         break;
1001     }
1002
1003     default:
1004         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1005         break;
1006     } 
1007 }
1008
1009 void SpeculativeJIT::compileIn(Node* node)
1010 {
1011     SpeculateCellOperand base(this, node->child1());
1012     GPRReg baseGPR = base.gpr();
1013     
1014     if (JSString* string = node->child2()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
1015         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1016             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1017             
1018             GPRTemporary result(this);
1019             GPRReg resultGPR = result.gpr();
1020
1021             use(node->child2());
1022             
1023             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1024             MacroAssembler::Label done = m_jit.label();
1025             
1026             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1027             // we can cast it to const AtomicStringImpl* safely.
1028             auto slowPath = slowPathCall(
1029                 jump.m_jump, this, operationInOptimize,
1030                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1031                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1032             
1033             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1034             stubInfo->codeOrigin = node->origin.semantic;
1035             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1036             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1037             stubInfo->patch.thisGPR = static_cast<int8_t>(InvalidGPRReg);
1038 #if USE(JSVALUE32_64)
1039             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1040             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1041             stubInfo->patch.thisTagGPR = static_cast<int8_t>(InvalidGPRReg);
1042 #endif
1043             stubInfo->patch.usedRegisters = usedRegisters();
1044
1045             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1046             addSlowPathGenerator(WTFMove(slowPath));
1047
1048             base.use();
1049
1050             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1051             return;
1052         }
1053     }
1054
1055     JSValueOperand key(this, node->child2());
1056     JSValueRegs regs = key.jsValueRegs();
1057         
1058     GPRFlushedCallResult result(this);
1059     GPRReg resultGPR = result.gpr();
1060         
1061     base.use();
1062     key.use();
1063         
1064     flushRegisters();
1065     callOperation(
1066         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1067         baseGPR, regs);
1068     m_jit.exceptionCheck();
1069     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1070 }
1071
1072 void SpeculativeJIT::compileDeleteById(Node* node)
1073 {
1074     JSValueOperand value(this, node->child1());
1075     GPRFlushedCallResult result(this);
1076
1077     JSValueRegs valueRegs = value.jsValueRegs();
1078     GPRReg resultGPR = result.gpr();
1079
1080     value.use();
1081
1082     flushRegisters();
1083     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1084     m_jit.exceptionCheck();
1085
1086     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1087 }
1088
1089 void SpeculativeJIT::compileDeleteByVal(Node* node)
1090 {
1091     JSValueOperand base(this, node->child1());
1092     JSValueOperand key(this, node->child2());
1093     GPRFlushedCallResult result(this);
1094
1095     JSValueRegs baseRegs = base.jsValueRegs();
1096     JSValueRegs keyRegs = key.jsValueRegs();
1097     GPRReg resultGPR = result.gpr();
1098
1099     base.use();
1100     key.use();
1101
1102     flushRegisters();
1103     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1104     m_jit.exceptionCheck();
1105
1106     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1107 }
1108
1109 void SpeculativeJIT::compilePushWithScope(Node* node)
1110 {
1111     SpeculateCellOperand currentScope(this, node->child1());
1112     GPRReg currentScopeGPR = currentScope.gpr();
1113
1114     GPRFlushedCallResult result(this);
1115     GPRReg resultGPR = result.gpr();
1116
1117     auto objectEdge = node->child2();
1118     if (objectEdge.useKind() == ObjectUse) {
1119         SpeculateCellOperand object(this, objectEdge);
1120         GPRReg objectGPR = object.gpr();
1121         speculateObject(objectEdge, objectGPR);
1122
1123         flushRegisters();
1124         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1125         // No exception check here as we did not have to call toObject().
1126     } else {
1127         ASSERT(objectEdge.useKind() == UntypedUse);
1128         JSValueOperand object(this, objectEdge);
1129         JSValueRegs objectRegs = object.jsValueRegs();
1130
1131         flushRegisters();
1132         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1133         m_jit.exceptionCheck();
1134     }
1135     
1136     cellResult(resultGPR, node);
1137 }
1138
1139 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1140 {
1141     unsigned branchIndexInBlock = detectPeepHoleBranch();
1142     if (branchIndexInBlock != UINT_MAX) {
1143         Node* branchNode = m_block->at(branchIndexInBlock);
1144
1145         ASSERT(node->adjustedRefCount() == 1);
1146         
1147         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1148     
1149         m_indexInBlock = branchIndexInBlock;
1150         m_currentNode = branchNode;
1151         
1152         return true;
1153     }
1154     
1155     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1156     
1157     return false;
1158 }
1159
1160 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1161 {
1162     unsigned branchIndexInBlock = detectPeepHoleBranch();
1163     if (branchIndexInBlock != UINT_MAX) {
1164         Node* branchNode = m_block->at(branchIndexInBlock);
1165
1166         ASSERT(node->adjustedRefCount() == 1);
1167         
1168         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1169     
1170         m_indexInBlock = branchIndexInBlock;
1171         m_currentNode = branchNode;
1172         
1173         return true;
1174     }
1175     
1176     nonSpeculativeNonPeepholeStrictEq(node, invert);
1177     
1178     return false;
1179 }
1180
1181 static const char* dataFormatString(DataFormat format)
1182 {
1183     // These values correspond to the DataFormat enum.
1184     const char* strings[] = {
1185         "[  ]",
1186         "[ i]",
1187         "[ d]",
1188         "[ c]",
1189         "Err!",
1190         "Err!",
1191         "Err!",
1192         "Err!",
1193         "[J ]",
1194         "[Ji]",
1195         "[Jd]",
1196         "[Jc]",
1197         "Err!",
1198         "Err!",
1199         "Err!",
1200         "Err!",
1201     };
1202     return strings[format];
1203 }
1204
1205 void SpeculativeJIT::dump(const char* label)
1206 {
1207     if (label)
1208         dataLogF("<%s>\n", label);
1209
1210     dataLogF("  gprs:\n");
1211     m_gprs.dump();
1212     dataLogF("  fprs:\n");
1213     m_fprs.dump();
1214     dataLogF("  VirtualRegisters:\n");
1215     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1216         GenerationInfo& info = m_generationInfo[i];
1217         if (info.alive())
1218             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1219         else
1220             dataLogF("    % 3d:[__][__]", i);
1221         if (info.registerFormat() == DataFormatDouble)
1222             dataLogF(":fpr%d\n", info.fpr());
1223         else if (info.registerFormat() != DataFormatNone
1224 #if USE(JSVALUE32_64)
1225             && !(info.registerFormat() & DataFormatJS)
1226 #endif
1227             ) {
1228             ASSERT(info.gpr() != InvalidGPRReg);
1229             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1230         } else
1231             dataLogF("\n");
1232     }
1233     if (label)
1234         dataLogF("</%s>\n", label);
1235 }
1236
1237 GPRTemporary::GPRTemporary()
1238     : m_jit(0)
1239     , m_gpr(InvalidGPRReg)
1240 {
1241 }
1242
1243 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1244     : m_jit(jit)
1245     , m_gpr(InvalidGPRReg)
1246 {
1247     m_gpr = m_jit->allocate();
1248 }
1249
1250 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1251     : m_jit(jit)
1252     , m_gpr(InvalidGPRReg)
1253 {
1254     m_gpr = m_jit->allocate(specific);
1255 }
1256
1257 #if USE(JSVALUE32_64)
1258 GPRTemporary::GPRTemporary(
1259     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1260     : m_jit(jit)
1261     , m_gpr(InvalidGPRReg)
1262 {
1263     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1264         m_gpr = m_jit->reuse(op1.gpr(which));
1265     else
1266         m_gpr = m_jit->allocate();
1267 }
1268 #endif // USE(JSVALUE32_64)
1269
1270 JSValueRegsTemporary::JSValueRegsTemporary() { }
1271
1272 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1273 #if USE(JSVALUE64)
1274     : m_gpr(jit)
1275 #else
1276     : m_payloadGPR(jit)
1277     , m_tagGPR(jit)
1278 #endif
1279 {
1280 }
1281
1282 #if USE(JSVALUE64)
1283 template<typename T>
1284 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1285     : m_gpr(jit, Reuse, operand)
1286 {
1287 }
1288 #else
1289 template<typename T>
1290 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1291 {
1292     if (resultWord == PayloadWord) {
1293         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1294         m_tagGPR = GPRTemporary(jit);
1295     } else {
1296         m_payloadGPR = GPRTemporary(jit);
1297         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1298     }
1299 }
1300 #endif
1301
1302 #if USE(JSVALUE64)
1303 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1304 {
1305     m_gpr = GPRTemporary(jit, Reuse, operand);
1306 }
1307 #else
1308 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1309 {
1310     if (jit->canReuse(operand.node())) {
1311         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1312         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1313     } else {
1314         m_payloadGPR = GPRTemporary(jit);
1315         m_tagGPR = GPRTemporary(jit);
1316     }
1317 }
1318 #endif
1319
1320 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1321
1322 JSValueRegs JSValueRegsTemporary::regs()
1323 {
1324 #if USE(JSVALUE64)
1325     return JSValueRegs(m_gpr.gpr());
1326 #else
1327     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1328 #endif
1329 }
1330
1331 void GPRTemporary::adopt(GPRTemporary& other)
1332 {
1333     ASSERT(!m_jit);
1334     ASSERT(m_gpr == InvalidGPRReg);
1335     ASSERT(other.m_jit);
1336     ASSERT(other.m_gpr != InvalidGPRReg);
1337     m_jit = other.m_jit;
1338     m_gpr = other.m_gpr;
1339     other.m_jit = 0;
1340     other.m_gpr = InvalidGPRReg;
1341 }
1342
1343 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1344 {
1345     ASSERT(other.m_jit);
1346     ASSERT(other.m_fpr != InvalidFPRReg);
1347     m_jit = other.m_jit;
1348     m_fpr = other.m_fpr;
1349
1350     other.m_jit = nullptr;
1351 }
1352
1353 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1354     : m_jit(jit)
1355     , m_fpr(InvalidFPRReg)
1356 {
1357     m_fpr = m_jit->fprAllocate();
1358 }
1359
1360 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1361     : m_jit(jit)
1362     , m_fpr(InvalidFPRReg)
1363 {
1364     if (m_jit->canReuse(op1.node()))
1365         m_fpr = m_jit->reuse(op1.fpr());
1366     else
1367         m_fpr = m_jit->fprAllocate();
1368 }
1369
1370 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1371     : m_jit(jit)
1372     , m_fpr(InvalidFPRReg)
1373 {
1374     if (m_jit->canReuse(op1.node()))
1375         m_fpr = m_jit->reuse(op1.fpr());
1376     else if (m_jit->canReuse(op2.node()))
1377         m_fpr = m_jit->reuse(op2.fpr());
1378     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1379         m_fpr = m_jit->reuse(op1.fpr());
1380     else
1381         m_fpr = m_jit->fprAllocate();
1382 }
1383
1384 #if USE(JSVALUE32_64)
1385 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1386     : m_jit(jit)
1387     , m_fpr(InvalidFPRReg)
1388 {
1389     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1390         m_fpr = m_jit->reuse(op1.fpr());
1391     else
1392         m_fpr = m_jit->fprAllocate();
1393 }
1394 #endif
1395
1396 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1397 {
1398     BasicBlock* taken = branchNode->branchData()->taken.block;
1399     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1400
1401     if (taken == nextBlock()) {
1402         condition = MacroAssembler::invert(condition);
1403         std::swap(taken, notTaken);
1404     }
1405
1406     SpeculateDoubleOperand op1(this, node->child1());
1407     SpeculateDoubleOperand op2(this, node->child2());
1408     
1409     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1410     jump(notTaken);
1411 }
1412
1413 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1414 {
1415     BasicBlock* taken = branchNode->branchData()->taken.block;
1416     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1417
1418     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1419     
1420     if (taken == nextBlock()) {
1421         condition = MacroAssembler::NotEqual;
1422         BasicBlock* tmp = taken;
1423         taken = notTaken;
1424         notTaken = tmp;
1425     }
1426
1427     SpeculateCellOperand op1(this, node->child1());
1428     SpeculateCellOperand op2(this, node->child2());
1429     
1430     GPRReg op1GPR = op1.gpr();
1431     GPRReg op2GPR = op2.gpr();
1432     
1433     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1434         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1435             speculationCheck(
1436                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1437         }
1438         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1439             speculationCheck(
1440                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1441         }
1442     } else {
1443         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1444             speculationCheck(
1445                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1446                 m_jit.branchIfNotObject(op1GPR));
1447         }
1448         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1449             m_jit.branchTest8(
1450                 MacroAssembler::NonZero, 
1451                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1452                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1453
1454         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1455             speculationCheck(
1456                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1457                 m_jit.branchIfNotObject(op2GPR));
1458         }
1459         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1460             m_jit.branchTest8(
1461                 MacroAssembler::NonZero, 
1462                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1463                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1464     }
1465
1466     branchPtr(condition, op1GPR, op2GPR, taken);
1467     jump(notTaken);
1468 }
1469
1470 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1471 {
1472     BasicBlock* taken = branchNode->branchData()->taken.block;
1473     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1474
1475     // The branch instruction will branch to the taken block.
1476     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1477     if (taken == nextBlock()) {
1478         condition = JITCompiler::invert(condition);
1479         BasicBlock* tmp = taken;
1480         taken = notTaken;
1481         notTaken = tmp;
1482     }
1483
1484     if (node->child1()->isInt32Constant()) {
1485         int32_t imm = node->child1()->asInt32();
1486         SpeculateBooleanOperand op2(this, node->child2());
1487         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1488     } else if (node->child2()->isInt32Constant()) {
1489         SpeculateBooleanOperand op1(this, node->child1());
1490         int32_t imm = node->child2()->asInt32();
1491         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1492     } else {
1493         SpeculateBooleanOperand op1(this, node->child1());
1494         SpeculateBooleanOperand op2(this, node->child2());
1495         branch32(condition, op1.gpr(), op2.gpr(), taken);
1496     }
1497
1498     jump(notTaken);
1499 }
1500
1501 void SpeculativeJIT::compileStringSlice(Node* node)
1502 {
1503     SpeculateCellOperand string(this, node->child1());
1504     GPRTemporary startIndex(this);
1505     GPRTemporary temp(this);
1506     GPRTemporary temp2(this);
1507
1508     GPRReg stringGPR = string.gpr();
1509     GPRReg startIndexGPR = startIndex.gpr();
1510     GPRReg tempGPR = temp.gpr();
1511     GPRReg temp2GPR = temp2.gpr();
1512
1513     speculateString(node->child1(), stringGPR);
1514
1515     {
1516         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1517
1518         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1519         if (node->child3())
1520             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1521         else
1522             m_jit.move(temp2GPR, tempGPR);
1523     }
1524
1525     CCallHelpers::JumpList doneCases;
1526     CCallHelpers::JumpList slowCases;
1527
1528     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1529     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1530     doneCases.append(m_jit.jump());
1531
1532     nonEmptyCase.link(&m_jit);
1533     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1534     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1535
1536     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1537     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1538
1539     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1540
1541     // Load the character into scratchReg
1542     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1543     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1544
1545     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1546     auto cont8Bit = m_jit.jump();
1547
1548     is16Bit.link(&m_jit);
1549     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1550
1551     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1552
1553     // 8 bit string values don't need the isASCII check.
1554     cont8Bit.link(&m_jit);
1555
1556     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1557     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1558     m_jit.loadPtr(tempGPR, tempGPR);
1559
1560     addSlowPathGenerator(
1561         slowPathCall(
1562             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1563
1564     addSlowPathGenerator(
1565         slowPathCall(
1566             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1567
1568     doneCases.link(&m_jit);
1569     cellResult(tempGPR, node);
1570 }
1571
1572 void SpeculativeJIT::compileToLowerCase(Node* node)
1573 {
1574     ASSERT(node->op() == ToLowerCase);
1575     SpeculateCellOperand string(this, node->child1());
1576     GPRTemporary temp(this);
1577     GPRTemporary index(this);
1578     GPRTemporary charReg(this);
1579     GPRTemporary length(this);
1580
1581     GPRReg stringGPR = string.gpr();
1582     GPRReg tempGPR = temp.gpr();
1583     GPRReg indexGPR = index.gpr();
1584     GPRReg charGPR = charReg.gpr();
1585     GPRReg lengthGPR = length.gpr();
1586
1587     speculateString(node->child1(), stringGPR);
1588
1589     CCallHelpers::JumpList slowPath;
1590
1591     m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1592
1593     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1594     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1595
1596     slowPath.append(m_jit.branchTest32(
1597         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1598         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1599     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1600     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1601
1602     auto loopStart = m_jit.label();
1603     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1604     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1605     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1606     m_jit.sub32(TrustedImm32('A'), charGPR);
1607     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1608
1609     m_jit.add32(TrustedImm32(1), indexGPR);
1610     m_jit.jump().linkTo(loopStart, &m_jit);
1611     
1612     slowPath.link(&m_jit);
1613     silentSpillAllRegisters(lengthGPR);
1614     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1615     silentFillAllRegisters();
1616     m_jit.exceptionCheck();
1617     auto done = m_jit.jump();
1618
1619     loopDone.link(&m_jit);
1620     m_jit.move(stringGPR, lengthGPR);
1621
1622     done.link(&m_jit);
1623     cellResult(lengthGPR, node);
1624 }
1625
1626 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1627 {
1628     BasicBlock* taken = branchNode->branchData()->taken.block;
1629     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1630
1631     // The branch instruction will branch to the taken block.
1632     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1633     if (taken == nextBlock()) {
1634         condition = JITCompiler::invert(condition);
1635         BasicBlock* tmp = taken;
1636         taken = notTaken;
1637         notTaken = tmp;
1638     }
1639
1640     if (node->child1()->isInt32Constant()) {
1641         int32_t imm = node->child1()->asInt32();
1642         SpeculateInt32Operand op2(this, node->child2());
1643         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1644     } else if (node->child2()->isInt32Constant()) {
1645         SpeculateInt32Operand op1(this, node->child1());
1646         int32_t imm = node->child2()->asInt32();
1647         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1648     } else {
1649         SpeculateInt32Operand op1(this, node->child1());
1650         SpeculateInt32Operand op2(this, node->child2());
1651         branch32(condition, op1.gpr(), op2.gpr(), taken);
1652     }
1653
1654     jump(notTaken);
1655 }
1656
1657 // Returns true if the compare is fused with a subsequent branch.
1658 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1659 {
1660     // Fused compare & branch.
1661     unsigned branchIndexInBlock = detectPeepHoleBranch();
1662     if (branchIndexInBlock != UINT_MAX) {
1663         Node* branchNode = m_block->at(branchIndexInBlock);
1664
1665         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1666         // so can be no intervening nodes to also reference the compare. 
1667         ASSERT(node->adjustedRefCount() == 1);
1668
1669         if (node->isBinaryUseKind(Int32Use))
1670             compilePeepHoleInt32Branch(node, branchNode, condition);
1671 #if USE(JSVALUE64)
1672         else if (node->isBinaryUseKind(Int52RepUse))
1673             compilePeepHoleInt52Branch(node, branchNode, condition);
1674 #endif // USE(JSVALUE64)
1675         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1676             // Use non-peephole comparison, for now.
1677             return false;
1678         } else if (node->isBinaryUseKind(DoubleRepUse))
1679             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1680         else if (node->op() == CompareEq) {
1681             if (node->isBinaryUseKind(BooleanUse))
1682                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1683             else if (node->isBinaryUseKind(SymbolUse))
1684                 compilePeepHoleSymbolEquality(node, branchNode);
1685             else if (node->isBinaryUseKind(ObjectUse))
1686                 compilePeepHoleObjectEquality(node, branchNode);
1687             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1688                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1689             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1690                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1691             else if (!needsTypeCheck(node->child1(), SpecOther))
1692                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1693             else if (!needsTypeCheck(node->child2(), SpecOther))
1694                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1695             else {
1696                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1697                 return true;
1698             }
1699         } else {
1700             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1701             return true;
1702         }
1703
1704         use(node->child1());
1705         use(node->child2());
1706         m_indexInBlock = branchIndexInBlock;
1707         m_currentNode = branchNode;
1708         return true;
1709     }
1710     return false;
1711 }
1712
1713 void SpeculativeJIT::noticeOSRBirth(Node* node)
1714 {
1715     if (!node->hasVirtualRegister())
1716         return;
1717     
1718     VirtualRegister virtualRegister = node->virtualRegister();
1719     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1720     
1721     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1722 }
1723
1724 void SpeculativeJIT::compileMovHint(Node* node)
1725 {
1726     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1727     
1728     Node* child = node->child1().node();
1729     noticeOSRBirth(child);
1730     
1731     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1732 }
1733
1734 void SpeculativeJIT::bail(AbortReason reason)
1735 {
1736     if (verboseCompilationEnabled())
1737         dataLog("Bailing compilation.\n");
1738     m_compileOkay = true;
1739     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1740     clearGenerationInfo();
1741 }
1742
1743 void SpeculativeJIT::compileCurrentBlock()
1744 {
1745     ASSERT(m_compileOkay);
1746     
1747     if (!m_block)
1748         return;
1749     
1750     ASSERT(m_block->isReachable);
1751     
1752     m_jit.blockHeads()[m_block->index] = m_jit.label();
1753
1754     if (!m_block->intersectionOfCFAHasVisited) {
1755         // Don't generate code for basic blocks that are unreachable according to CFA.
1756         // But to be sure that nobody has generated a jump to this block, drop in a
1757         // breakpoint here.
1758         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1759         return;
1760     }
1761
1762     if (m_block->isCatchEntrypoint) {
1763         m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1764         if (Options::zeroStackFrame())
1765             m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1766         m_jit.emitSaveCalleeSaves();
1767         m_jit.emitMaterializeTagCheckRegisters();
1768         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1769     }
1770
1771     m_stream->appendAndLog(VariableEvent::reset());
1772     
1773     m_jit.jitAssertHasValidCallFrame();
1774     m_jit.jitAssertTagsInPlace();
1775     m_jit.jitAssertArgumentCountSane();
1776
1777     m_state.reset();
1778     m_state.beginBasicBlock(m_block);
1779     
1780     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1781         int operand = m_block->variablesAtHead.operandForIndex(i);
1782         Node* node = m_block->variablesAtHead[i];
1783         if (!node)
1784             continue; // No need to record dead SetLocal's.
1785         
1786         VariableAccessData* variable = node->variableAccessData();
1787         DataFormat format;
1788         if (!node->refCount())
1789             continue; // No need to record dead SetLocal's.
1790         format = dataFormatFor(variable->flushFormat());
1791         m_stream->appendAndLog(
1792             VariableEvent::setLocal(
1793                 VirtualRegister(operand),
1794                 variable->machineLocal(),
1795                 format));
1796     }
1797
1798     m_origin = NodeOrigin();
1799     
1800     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1801         m_currentNode = m_block->at(m_indexInBlock);
1802         
1803         // We may have hit a contradiction that the CFA was aware of but that the JIT
1804         // didn't cause directly.
1805         if (!m_state.isValid()) {
1806             bail(DFGBailedAtTopOfBlock);
1807             return;
1808         }
1809
1810         m_interpreter.startExecuting();
1811         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1812         m_jit.setForNode(m_currentNode);
1813         m_origin = m_currentNode->origin;
1814         if (validationEnabled())
1815             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1816         m_lastGeneratedNode = m_currentNode->op();
1817         
1818         ASSERT(m_currentNode->shouldGenerate());
1819         
1820         if (verboseCompilationEnabled()) {
1821             dataLogF(
1822                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1823                 (int)m_currentNode->index(),
1824                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1825             dataLog("\n");
1826         }
1827
1828         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1829             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1830
1831         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1832
1833         compile(m_currentNode);
1834         
1835         if (belongsInMinifiedGraph(m_currentNode->op()))
1836             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1837         
1838 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1839         m_jit.clearRegisterAllocationOffsets();
1840 #endif
1841         
1842         if (!m_compileOkay) {
1843             bail(DFGBailedAtEndOfNode);
1844             return;
1845         }
1846         
1847         // Make sure that the abstract state is rematerialized for the next node.
1848         m_interpreter.executeEffects(m_indexInBlock);
1849     }
1850     
1851     // Perform the most basic verification that children have been used correctly.
1852     if (!ASSERT_DISABLED) {
1853         for (auto& info : m_generationInfo)
1854             RELEASE_ASSERT(!info.alive());
1855     }
1856 }
1857
1858 // If we are making type predictions about our arguments then
1859 // we need to check that they are correct on function entry.
1860 void SpeculativeJIT::checkArgumentTypes()
1861 {
1862     ASSERT(!m_currentNode);
1863     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1864
1865     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1866     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1867         Node* node = arguments[i];
1868         if (!node) {
1869             // The argument is dead. We don't do any checks for such arguments.
1870             continue;
1871         }
1872         
1873         ASSERT(node->op() == SetArgument);
1874         ASSERT(node->shouldGenerate());
1875
1876         VariableAccessData* variableAccessData = node->variableAccessData();
1877         FlushFormat format = variableAccessData->flushFormat();
1878         
1879         if (format == FlushedJSValue)
1880             continue;
1881         
1882         VirtualRegister virtualRegister = variableAccessData->local();
1883
1884         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1885         
1886 #if USE(JSVALUE64)
1887         switch (format) {
1888         case FlushedInt32: {
1889             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1890             break;
1891         }
1892         case FlushedBoolean: {
1893             GPRTemporary temp(this);
1894             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1895             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1896             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1897             break;
1898         }
1899         case FlushedCell: {
1900             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1901             break;
1902         }
1903         default:
1904             RELEASE_ASSERT_NOT_REACHED();
1905             break;
1906         }
1907 #else
1908         switch (format) {
1909         case FlushedInt32: {
1910             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1911             break;
1912         }
1913         case FlushedBoolean: {
1914             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1915             break;
1916         }
1917         case FlushedCell: {
1918             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1919             break;
1920         }
1921         default:
1922             RELEASE_ASSERT_NOT_REACHED();
1923             break;
1924         }
1925 #endif
1926     }
1927
1928     m_origin = NodeOrigin();
1929 }
1930
1931 bool SpeculativeJIT::compile()
1932 {
1933     checkArgumentTypes();
1934     
1935     ASSERT(!m_currentNode);
1936     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1937         m_jit.setForBlockIndex(blockIndex);
1938         m_block = m_jit.graph().block(blockIndex);
1939         compileCurrentBlock();
1940     }
1941     linkBranches();
1942     return true;
1943 }
1944
1945 void SpeculativeJIT::createOSREntries()
1946 {
1947     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1948         BasicBlock* block = m_jit.graph().block(blockIndex);
1949         if (!block)
1950             continue;
1951         if (block->isOSRTarget || block->isCatchEntrypoint) {
1952             // Currently we don't have OSR entry trampolines. We could add them
1953             // here if need be.
1954             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1955         }
1956     }
1957 }
1958
1959 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1960 {
1961     unsigned osrEntryIndex = 0;
1962     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1963         BasicBlock* block = m_jit.graph().block(blockIndex);
1964         if (!block)
1965             continue;
1966         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1967             continue;
1968         if (block->isCatchEntrypoint) {
1969             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1970             Vector<FlushFormat> argumentFormats;
1971             argumentFormats.reserveInitialCapacity(argumentsVector.size());
1972             for (Node* setArgument : argumentsVector) {
1973                 if (setArgument) {
1974                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
1975                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
1976                     argumentFormats.uncheckedAppend(flushFormat);
1977                 } else
1978                     argumentFormats.uncheckedAppend(DeadFlush);
1979             }
1980             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
1981         } else {
1982             ASSERT(block->isOSRTarget);
1983             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
1984         }
1985     }
1986
1987     m_jit.jitCode()->finalizeOSREntrypoints();
1988     m_jit.jitCode()->common.finalizeCatchEntrypoints();
1989
1990     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
1991     
1992     if (verboseCompilationEnabled()) {
1993         DumpContext dumpContext;
1994         dataLog("OSR Entries:\n");
1995         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
1996             dataLog("    ", inContext(entryData, &dumpContext), "\n");
1997         if (!dumpContext.isEmpty())
1998             dumpContext.dump(WTF::dataFile());
1999     }
2000 }
2001     
2002 void SpeculativeJIT::compileCheckTraps(Node*)
2003 {
2004     ASSERT(Options::usePollingTraps());
2005     GPRTemporary unused(this);
2006     GPRReg unusedGPR = unused.gpr();
2007
2008     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2009         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2010
2011     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2012 }
2013
2014 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2015 {
2016     Edge child3 = m_jit.graph().varArgChild(node, 2);
2017     Edge child4 = m_jit.graph().varArgChild(node, 3);
2018
2019     ArrayMode arrayMode = node->arrayMode();
2020     
2021     GPRReg baseReg = base.gpr();
2022     GPRReg propertyReg = property.gpr();
2023     
2024     SpeculateDoubleOperand value(this, child3);
2025
2026     FPRReg valueReg = value.fpr();
2027     
2028     DFG_TYPE_CHECK(
2029         JSValueRegs(), child3, SpecFullRealNumber,
2030         m_jit.branchDouble(
2031             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2032     
2033     if (!m_compileOkay)
2034         return;
2035     
2036     StorageOperand storage(this, child4);
2037     GPRReg storageReg = storage.gpr();
2038
2039     if (node->op() == PutByValAlias) {
2040         // Store the value to the array.
2041         GPRReg propertyReg = property.gpr();
2042         FPRReg valueReg = value.fpr();
2043         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2044         
2045         noResult(m_currentNode);
2046         return;
2047     }
2048     
2049     GPRTemporary temporary;
2050     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2051
2052     MacroAssembler::Jump slowCase;
2053     
2054     if (arrayMode.isInBounds()) {
2055         speculationCheck(
2056             OutOfBounds, JSValueRegs(), 0,
2057             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2058     } else {
2059         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2060         
2061         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2062         
2063         if (!arrayMode.isOutOfBounds())
2064             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2065         
2066         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2067         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2068
2069         inBounds.link(&m_jit);
2070     }
2071     
2072     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2073
2074     base.use();
2075     property.use();
2076     value.use();
2077     storage.use();
2078     
2079     if (arrayMode.isOutOfBounds()) {
2080         addSlowPathGenerator(
2081             slowPathCall(
2082                 slowCase, this,
2083                 m_jit.codeBlock()->isStrictMode()
2084                     ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2085                     : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2086                 NoResult, baseReg, propertyReg, valueReg));
2087     }
2088
2089     noResult(m_currentNode, UseChildrenCalledExplicitly);
2090 }
2091
2092 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2093 {
2094     SpeculateCellOperand string(this, node->child1());
2095     SpeculateStrictInt32Operand index(this, node->child2());
2096     StorageOperand storage(this, node->child3());
2097
2098     GPRReg stringReg = string.gpr();
2099     GPRReg indexReg = index.gpr();
2100     GPRReg storageReg = storage.gpr();
2101     
2102     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2103
2104     // unsigned comparison so we can filter out negative indices and indices that are too large
2105     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2106
2107     GPRTemporary scratch(this);
2108     GPRReg scratchReg = scratch.gpr();
2109
2110     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2111
2112     // Load the character into scratchReg
2113     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2114
2115     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2116     JITCompiler::Jump cont8Bit = m_jit.jump();
2117
2118     is16Bit.link(&m_jit);
2119
2120     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2121
2122     cont8Bit.link(&m_jit);
2123
2124     int32Result(scratchReg, m_currentNode);
2125 }
2126
2127 void SpeculativeJIT::compileGetByValOnString(Node* node)
2128 {
2129     SpeculateCellOperand base(this, m_graph.child(node, 0));
2130     SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2131     StorageOperand storage(this, m_graph.child(node, 2));
2132     GPRReg baseReg = base.gpr();
2133     GPRReg propertyReg = property.gpr();
2134     GPRReg storageReg = storage.gpr();
2135
2136     GPRTemporary scratch(this);
2137     GPRReg scratchReg = scratch.gpr();
2138 #if USE(JSVALUE32_64)
2139     GPRTemporary resultTag;
2140     GPRReg resultTagReg = InvalidGPRReg;
2141     if (node->arrayMode().isOutOfBounds()) {
2142         GPRTemporary realResultTag(this);
2143         resultTag.adopt(realResultTag);
2144         resultTagReg = resultTag.gpr();
2145     }
2146 #endif
2147
2148     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2149
2150     // unsigned comparison so we can filter out negative indices and indices that are too large
2151     JITCompiler::Jump outOfBounds = m_jit.branch32(
2152         MacroAssembler::AboveOrEqual, propertyReg,
2153         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2154     if (node->arrayMode().isInBounds())
2155         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2156
2157     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2158
2159     // Load the character into scratchReg
2160     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2161
2162     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2163     JITCompiler::Jump cont8Bit = m_jit.jump();
2164
2165     is16Bit.link(&m_jit);
2166
2167     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2168
2169     JITCompiler::Jump bigCharacter =
2170         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2171
2172     // 8 bit string values don't need the isASCII check.
2173     cont8Bit.link(&m_jit);
2174
2175     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2176     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2177     m_jit.loadPtr(scratchReg, scratchReg);
2178
2179     addSlowPathGenerator(
2180         slowPathCall(
2181             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2182
2183     if (node->arrayMode().isOutOfBounds()) {
2184 #if USE(JSVALUE32_64)
2185         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2186 #endif
2187
2188         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2189         bool prototypeChainIsSane = false;
2190         if (globalObject->stringPrototypeChainIsSane()) {
2191             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2192             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2193             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2194             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2195             // indexed properties either.
2196             // https://bugs.webkit.org/show_bug.cgi?id=144668
2197             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure());
2198             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure());
2199             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2200         }
2201         if (prototypeChainIsSane) {
2202 #if USE(JSVALUE64)
2203             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2204                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2205 #else
2206             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2207                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2208                 baseReg, propertyReg));
2209 #endif
2210         } else {
2211 #if USE(JSVALUE64)
2212             addSlowPathGenerator(
2213                 slowPathCall(
2214                     outOfBounds, this, operationGetByValStringInt,
2215                     scratchReg, baseReg, propertyReg));
2216 #else
2217             addSlowPathGenerator(
2218                 slowPathCall(
2219                     outOfBounds, this, operationGetByValStringInt,
2220                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2221 #endif
2222         }
2223         
2224 #if USE(JSVALUE64)
2225         jsValueResult(scratchReg, m_currentNode);
2226 #else
2227         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2228 #endif
2229     } else
2230         cellResult(scratchReg, m_currentNode);
2231 }
2232
2233 void SpeculativeJIT::compileFromCharCode(Node* node)
2234 {
2235     Edge& child = node->child1();
2236     if (child.useKind() == UntypedUse) {
2237         JSValueOperand opr(this, child);
2238         JSValueRegs oprRegs = opr.jsValueRegs();
2239
2240         flushRegisters();
2241         JSValueRegsFlushedCallResult result(this);
2242         JSValueRegs resultRegs = result.regs();
2243         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2244         m_jit.exceptionCheck();
2245         
2246         jsValueResult(resultRegs, node);
2247         return;
2248     }
2249
2250     SpeculateStrictInt32Operand property(this, child);
2251     GPRReg propertyReg = property.gpr();
2252     GPRTemporary smallStrings(this);
2253     GPRTemporary scratch(this);
2254     GPRReg scratchReg = scratch.gpr();
2255     GPRReg smallStringsReg = smallStrings.gpr();
2256
2257     JITCompiler::JumpList slowCases;
2258     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2259     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2260     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2261
2262     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2263     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2264     cellResult(scratchReg, m_currentNode);
2265 }
2266
2267 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2268 {
2269     VirtualRegister virtualRegister = node->virtualRegister();
2270     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2271
2272     switch (info.registerFormat()) {
2273     case DataFormatStorage:
2274         RELEASE_ASSERT_NOT_REACHED();
2275
2276     case DataFormatBoolean:
2277     case DataFormatCell:
2278         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2279         return GeneratedOperandTypeUnknown;
2280
2281     case DataFormatNone:
2282     case DataFormatJSCell:
2283     case DataFormatJS:
2284     case DataFormatJSBoolean:
2285     case DataFormatJSDouble:
2286         return GeneratedOperandJSValue;
2287
2288     case DataFormatJSInt32:
2289     case DataFormatInt32:
2290         return GeneratedOperandInteger;
2291
2292     default:
2293         RELEASE_ASSERT_NOT_REACHED();
2294         return GeneratedOperandTypeUnknown;
2295     }
2296 }
2297
2298 void SpeculativeJIT::compileValueToInt32(Node* node)
2299 {
2300     switch (node->child1().useKind()) {
2301 #if USE(JSVALUE64)
2302     case Int52RepUse: {
2303         SpeculateStrictInt52Operand op1(this, node->child1());
2304         GPRTemporary result(this, Reuse, op1);
2305         GPRReg op1GPR = op1.gpr();
2306         GPRReg resultGPR = result.gpr();
2307         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2308         int32Result(resultGPR, node, DataFormatInt32);
2309         return;
2310     }
2311 #endif // USE(JSVALUE64)
2312         
2313     case DoubleRepUse: {
2314         GPRTemporary result(this);
2315         SpeculateDoubleOperand op1(this, node->child1());
2316         FPRReg fpr = op1.fpr();
2317         GPRReg gpr = result.gpr();
2318         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2319         
2320         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2321             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2322         
2323         int32Result(gpr, node);
2324         return;
2325     }
2326     
2327     case NumberUse:
2328     case NotCellUse: {
2329         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2330         case GeneratedOperandInteger: {
2331             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2332             GPRTemporary result(this, Reuse, op1);
2333             m_jit.move(op1.gpr(), result.gpr());
2334             int32Result(result.gpr(), node, op1.format());
2335             return;
2336         }
2337         case GeneratedOperandJSValue: {
2338             GPRTemporary result(this);
2339 #if USE(JSVALUE64)
2340             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2341
2342             GPRReg gpr = op1.gpr();
2343             GPRReg resultGpr = result.gpr();
2344             FPRTemporary tempFpr(this);
2345             FPRReg fpr = tempFpr.fpr();
2346
2347             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2348             JITCompiler::JumpList converted;
2349
2350             if (node->child1().useKind() == NumberUse) {
2351                 DFG_TYPE_CHECK(
2352                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2353                     m_jit.branchTest64(
2354                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2355             } else {
2356                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2357                 
2358                 DFG_TYPE_CHECK(
2359                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2360                 
2361                 // It's not a cell: so true turns into 1 and all else turns into 0.
2362                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2363                 converted.append(m_jit.jump());
2364                 
2365                 isNumber.link(&m_jit);
2366             }
2367
2368             // First, if we get here we have a double encoded as a JSValue
2369             unboxDouble(gpr, resultGpr, fpr);
2370
2371             silentSpillAllRegisters(resultGpr);
2372             callOperation(operationToInt32, resultGpr, fpr);
2373             silentFillAllRegisters();
2374
2375             converted.append(m_jit.jump());
2376
2377             isInteger.link(&m_jit);
2378             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2379
2380             converted.link(&m_jit);
2381 #else
2382             Node* childNode = node->child1().node();
2383             VirtualRegister virtualRegister = childNode->virtualRegister();
2384             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2385
2386             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2387
2388             GPRReg payloadGPR = op1.payloadGPR();
2389             GPRReg resultGpr = result.gpr();
2390         
2391             JITCompiler::JumpList converted;
2392
2393             if (info.registerFormat() == DataFormatJSInt32)
2394                 m_jit.move(payloadGPR, resultGpr);
2395             else {
2396                 GPRReg tagGPR = op1.tagGPR();
2397                 FPRTemporary tempFpr(this);
2398                 FPRReg fpr = tempFpr.fpr();
2399                 FPRTemporary scratch(this);
2400
2401                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2402
2403                 if (node->child1().useKind() == NumberUse) {
2404                     DFG_TYPE_CHECK(
2405                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2406                         m_jit.branch32(
2407                             MacroAssembler::AboveOrEqual, tagGPR,
2408                             TrustedImm32(JSValue::LowestTag)));
2409                 } else {
2410                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2411                     
2412                     DFG_TYPE_CHECK(
2413                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2414                         m_jit.branchIfCell(op1.jsValueRegs()));
2415                     
2416                     // It's not a cell: so true turns into 1 and all else turns into 0.
2417                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2418                     m_jit.move(TrustedImm32(0), resultGpr);
2419                     converted.append(m_jit.jump());
2420                     
2421                     isBoolean.link(&m_jit);
2422                     m_jit.move(payloadGPR, resultGpr);
2423                     converted.append(m_jit.jump());
2424                     
2425                     isNumber.link(&m_jit);
2426                 }
2427
2428                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2429
2430                 silentSpillAllRegisters(resultGpr);
2431                 callOperation(operationToInt32, resultGpr, fpr);
2432                 silentFillAllRegisters();
2433
2434                 converted.append(m_jit.jump());
2435
2436                 isInteger.link(&m_jit);
2437                 m_jit.move(payloadGPR, resultGpr);
2438
2439                 converted.link(&m_jit);
2440             }
2441 #endif
2442             int32Result(resultGpr, node);
2443             return;
2444         }
2445         case GeneratedOperandTypeUnknown:
2446             RELEASE_ASSERT(!m_compileOkay);
2447             return;
2448         }
2449         RELEASE_ASSERT_NOT_REACHED();
2450         return;
2451     }
2452     
2453     default:
2454         ASSERT(!m_compileOkay);
2455         return;
2456     }
2457 }
2458
2459 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2460 {
2461     if (doesOverflow(node->arithMode())) {
2462         if (enableInt52()) {
2463             SpeculateInt32Operand op1(this, node->child1());
2464             GPRTemporary result(this, Reuse, op1);
2465             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2466             strictInt52Result(result.gpr(), node);
2467             return;
2468         }
2469         SpeculateInt32Operand op1(this, node->child1());
2470         FPRTemporary result(this);
2471             
2472         GPRReg inputGPR = op1.gpr();
2473         FPRReg outputFPR = result.fpr();
2474             
2475         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2476             
2477         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2478         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2479         positive.link(&m_jit);
2480             
2481         doubleResult(outputFPR, node);
2482         return;
2483     }
2484     
2485     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2486
2487     SpeculateInt32Operand op1(this, node->child1());
2488     GPRTemporary result(this);
2489
2490     m_jit.move(op1.gpr(), result.gpr());
2491
2492     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2493
2494     int32Result(result.gpr(), node, op1.format());
2495 }
2496
2497 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2498 {
2499     SpeculateDoubleOperand op1(this, node->child1());
2500     FPRTemporary scratch(this);
2501     GPRTemporary result(this);
2502     
2503     FPRReg valueFPR = op1.fpr();
2504     FPRReg scratchFPR = scratch.fpr();
2505     GPRReg resultGPR = result.gpr();
2506
2507     JITCompiler::JumpList failureCases;
2508     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2509     m_jit.branchConvertDoubleToInt32(
2510         valueFPR, resultGPR, failureCases, scratchFPR,
2511         shouldCheckNegativeZero(node->arithMode()));
2512     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2513
2514     int32Result(resultGPR, node);
2515 }
2516
2517 void SpeculativeJIT::compileDoubleRep(Node* node)
2518 {
2519     switch (node->child1().useKind()) {
2520     case RealNumberUse: {
2521         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2522         FPRTemporary result(this);
2523         
2524         JSValueRegs op1Regs = op1.jsValueRegs();
2525         FPRReg resultFPR = result.fpr();
2526         
2527 #if USE(JSVALUE64)
2528         GPRTemporary temp(this);
2529         GPRReg tempGPR = temp.gpr();
2530         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2531 #else
2532         FPRTemporary temp(this);
2533         FPRReg tempFPR = temp.fpr();
2534         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2535 #endif
2536         
2537         JITCompiler::Jump done = m_jit.branchDouble(
2538             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2539         
2540         DFG_TYPE_CHECK(
2541             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2542         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2543         
2544         done.link(&m_jit);
2545         
2546         doubleResult(resultFPR, node);
2547         return;
2548     }
2549     
2550     case NotCellUse:
2551     case NumberUse: {
2552         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2553
2554         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2555         if (isInt32Speculation(possibleTypes)) {
2556             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2557             FPRTemporary result(this);
2558             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2559             doubleResult(result.fpr(), node);
2560             return;
2561         }
2562
2563         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2564         FPRTemporary result(this);
2565
2566 #if USE(JSVALUE64)
2567         GPRTemporary temp(this);
2568
2569         GPRReg op1GPR = op1.gpr();
2570         GPRReg tempGPR = temp.gpr();
2571         FPRReg resultFPR = result.fpr();
2572         JITCompiler::JumpList done;
2573
2574         JITCompiler::Jump isInteger = m_jit.branch64(
2575             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2576
2577         if (node->child1().useKind() == NotCellUse) {
2578             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2579             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2580
2581             static const double zero = 0;
2582             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2583
2584             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2585             done.append(isNull);
2586
2587             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2588                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2589
2590             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2591             static const double one = 1;
2592             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2593             done.append(m_jit.jump());
2594             done.append(isFalse);
2595
2596             isUndefined.link(&m_jit);
2597             static const double NaN = PNaN;
2598             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2599             done.append(m_jit.jump());
2600
2601             isNumber.link(&m_jit);
2602         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2603             typeCheck(
2604                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2605                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2606         }
2607
2608         unboxDouble(op1GPR, tempGPR, resultFPR);
2609         done.append(m_jit.jump());
2610     
2611         isInteger.link(&m_jit);
2612         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2613         done.link(&m_jit);
2614 #else // USE(JSVALUE64) -> this is the 32_64 case
2615         FPRTemporary temp(this);
2616     
2617         GPRReg op1TagGPR = op1.tagGPR();
2618         GPRReg op1PayloadGPR = op1.payloadGPR();
2619         FPRReg tempFPR = temp.fpr();
2620         FPRReg resultFPR = result.fpr();
2621         JITCompiler::JumpList done;
2622     
2623         JITCompiler::Jump isInteger = m_jit.branch32(
2624             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2625
2626         if (node->child1().useKind() == NotCellUse) {
2627             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2628             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2629
2630             static const double zero = 0;
2631             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2632
2633             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2634             done.append(isNull);
2635
2636             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2637
2638             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2639             static const double one = 1;
2640             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2641             done.append(m_jit.jump());
2642             done.append(isFalse);
2643
2644             isUndefined.link(&m_jit);
2645             static const double NaN = PNaN;
2646             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2647             done.append(m_jit.jump());
2648
2649             isNumber.link(&m_jit);
2650         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2651             typeCheck(
2652                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2653                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2654         }
2655
2656         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2657         done.append(m_jit.jump());
2658     
2659         isInteger.link(&m_jit);
2660         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2661         done.link(&m_jit);
2662 #endif // USE(JSVALUE64)
2663     
2664         doubleResult(resultFPR, node);
2665         return;
2666     }
2667         
2668 #if USE(JSVALUE64)
2669     case Int52RepUse: {
2670         SpeculateStrictInt52Operand value(this, node->child1());
2671         FPRTemporary result(this);
2672         
2673         GPRReg valueGPR = value.gpr();
2674         FPRReg resultFPR = result.fpr();
2675
2676         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2677         
2678         doubleResult(resultFPR, node);
2679         return;
2680     }
2681 #endif // USE(JSVALUE64)
2682         
2683     default:
2684         RELEASE_ASSERT_NOT_REACHED();
2685         return;
2686     }
2687 }
2688
2689 void SpeculativeJIT::compileValueRep(Node* node)
2690 {
2691     switch (node->child1().useKind()) {
2692     case DoubleRepUse: {
2693         SpeculateDoubleOperand value(this, node->child1());
2694         JSValueRegsTemporary result(this);
2695         
2696         FPRReg valueFPR = value.fpr();
2697         JSValueRegs resultRegs = result.regs();
2698         
2699         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2700         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2701         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2702         // local was purified.
2703         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2704             m_jit.purifyNaN(valueFPR);
2705
2706         boxDouble(valueFPR, resultRegs);
2707         
2708         jsValueResult(resultRegs, node);
2709         return;
2710     }
2711         
2712 #if USE(JSVALUE64)
2713     case Int52RepUse: {
2714         SpeculateStrictInt52Operand value(this, node->child1());
2715         GPRTemporary result(this);
2716         
2717         GPRReg valueGPR = value.gpr();
2718         GPRReg resultGPR = result.gpr();
2719         
2720         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2721         
2722         jsValueResult(resultGPR, node);
2723         return;
2724     }
2725 #endif // USE(JSVALUE64)
2726         
2727     default:
2728         RELEASE_ASSERT_NOT_REACHED();
2729         return;
2730     }
2731 }
2732
2733 static double clampDoubleToByte(double d)
2734 {
2735     d += 0.5;
2736     if (!(d > 0))
2737         d = 0;
2738     else if (d > 255)
2739         d = 255;
2740     return d;
2741 }
2742
2743 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2744 {
2745     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2746     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2747     jit.xorPtr(result, result);
2748     MacroAssembler::Jump clamped = jit.jump();
2749     tooBig.link(&jit);
2750     jit.move(JITCompiler::TrustedImm32(255), result);
2751     clamped.link(&jit);
2752     inBounds.link(&jit);
2753 }
2754
2755 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2756 {
2757     // Unordered compare so we pick up NaN
2758     static const double zero = 0;
2759     static const double byteMax = 255;
2760     static const double half = 0.5;
2761     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2762     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2763     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2764     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2765     
2766     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2767     // FIXME: This should probably just use a floating point round!
2768     // https://bugs.webkit.org/show_bug.cgi?id=72054
2769     jit.addDouble(source, scratch);
2770     jit.truncateDoubleToInt32(scratch, result);   
2771     MacroAssembler::Jump truncatedInt = jit.jump();
2772     
2773     tooSmall.link(&jit);
2774     jit.xorPtr(result, result);
2775     MacroAssembler::Jump zeroed = jit.jump();
2776     
2777     tooBig.link(&jit);
2778     jit.move(JITCompiler::TrustedImm32(255), result);
2779     
2780     truncatedInt.link(&jit);
2781     zeroed.link(&jit);
2782
2783 }
2784
2785 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2786 {
2787     if (node->op() == PutByValAlias)
2788         return JITCompiler::Jump();
2789     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2790         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2791     if (view) {
2792         uint32_t length = view->length();
2793         Node* indexNode = m_jit.graph().child(node, 1).node();
2794         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2795             return JITCompiler::Jump();
2796         return m_jit.branch32(
2797             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2798     }
2799     return m_jit.branch32(
2800         MacroAssembler::AboveOrEqual, indexGPR,
2801         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2802 }
2803
2804 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2805 {
2806     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2807     if (!jump.isSet())
2808         return;
2809     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2810 }
2811
2812 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2813 {
2814     JITCompiler::Jump done;
2815     if (outOfBounds.isSet()) {
2816         done = m_jit.jump();
2817         if (node->arrayMode().isInBounds())
2818             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2819         else {
2820             outOfBounds.link(&m_jit);
2821
2822             JITCompiler::Jump notWasteful = m_jit.branch32(
2823                 MacroAssembler::NotEqual,
2824                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2825                 TrustedImm32(WastefulTypedArray));
2826
2827             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2828                 MacroAssembler::Zero,
2829                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfPoisonedVector()));
2830             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2831             notWasteful.link(&m_jit);
2832         }
2833     }
2834     return done;
2835 }
2836
2837 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2838 {
2839     switch (elementSize(type)) {
2840     case 1:
2841         if (isSigned(type))
2842             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2843         else
2844             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2845         break;
2846     case 2:
2847         if (isSigned(type))
2848             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2849         else
2850             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2851         break;
2852     case 4:
2853         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2854         break;
2855     default:
2856         CRASH();
2857     }
2858 }
2859
2860 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2861 {
2862     if (elementSize(type) < 4 || isSigned(type)) {
2863         int32Result(resultReg, node);
2864         return;
2865     }
2866     
2867     ASSERT(elementSize(type) == 4 && !isSigned(type));
2868     if (node->shouldSpeculateInt32() && canSpeculate) {
2869         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2870         int32Result(resultReg, node);
2871         return;
2872     }
2873     
2874 #if USE(JSVALUE64)
2875     if (node->shouldSpeculateAnyInt()) {
2876         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2877         strictInt52Result(resultReg, node);
2878         return;
2879     }
2880 #endif
2881     
2882     FPRTemporary fresult(this);
2883     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2884     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2885     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2886     positive.link(&m_jit);
2887     doubleResult(fresult.fpr(), node);
2888 }
2889
2890 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2891 {
2892     ASSERT(isInt(type));
2893     
2894     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2895     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2896     StorageOperand storage(this, m_graph.varArgChild(node, 2));
2897
2898     GPRReg baseReg = base.gpr();
2899     GPRReg propertyReg = property.gpr();
2900     GPRReg storageReg = storage.gpr();
2901
2902     GPRTemporary result(this);
2903     GPRReg resultReg = result.gpr();
2904
2905     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2906
2907     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2908     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2909     bool canSpeculate = true;
2910     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2911 }
2912
2913 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2914     GPRTemporary& value,
2915     GPRReg property,
2916 #if USE(JSVALUE32_64)
2917     GPRTemporary& propertyTag,
2918     GPRTemporary& valueTag,
2919 #endif
2920     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2921 {
2922     bool isAppropriateConstant = false;
2923     if (valueUse->isConstant()) {
2924         JSValue jsValue = valueUse->asJSValue();
2925         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2926         SpeculatedType actualType = speculationFromValue(jsValue);
2927         isAppropriateConstant = (expectedType | actualType) == expectedType;
2928     }
2929     
2930     if (isAppropriateConstant) {
2931         JSValue jsValue = valueUse->asJSValue();
2932         if (!jsValue.isNumber()) {
2933             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2934             return false;
2935         }
2936         double d = jsValue.asNumber();
2937         if (isClamped)
2938             d = clampDoubleToByte(d);
2939         GPRTemporary scratch(this);
2940         GPRReg scratchReg = scratch.gpr();
2941         m_jit.move(Imm32(toInt32(d)), scratchReg);
2942         value.adopt(scratch);
2943     } else {
2944         switch (valueUse.useKind()) {
2945         case Int32Use: {
2946             SpeculateInt32Operand valueOp(this, valueUse);
2947             GPRTemporary scratch(this);
2948             GPRReg scratchReg = scratch.gpr();
2949             m_jit.move(valueOp.gpr(), scratchReg);
2950             if (isClamped)
2951                 compileClampIntegerToByte(m_jit, scratchReg);
2952             value.adopt(scratch);
2953             break;
2954         }
2955             
2956 #if USE(JSVALUE64)
2957         case Int52RepUse: {
2958             SpeculateStrictInt52Operand valueOp(this, valueUse);
2959             GPRTemporary scratch(this);
2960             GPRReg scratchReg = scratch.gpr();
2961             m_jit.move(valueOp.gpr(), scratchReg);
2962             if (isClamped) {
2963                 MacroAssembler::Jump inBounds = m_jit.branch64(
2964                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2965                 MacroAssembler::Jump tooBig = m_jit.branch64(
2966                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2967                 m_jit.move(TrustedImm32(0), scratchReg);
2968                 MacroAssembler::Jump clamped = m_jit.jump();
2969                 tooBig.link(&m_jit);
2970                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2971                 clamped.link(&m_jit);
2972                 inBounds.link(&m_jit);
2973             }
2974             value.adopt(scratch);
2975             break;
2976         }
2977 #endif // USE(JSVALUE64)
2978             
2979         case DoubleRepUse: {
2980             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
2981             if (isClamped) {
2982                 SpeculateDoubleOperand valueOp(this, valueUse);
2983                 GPRTemporary result(this);
2984                 FPRTemporary floatScratch(this);
2985                 FPRReg fpr = valueOp.fpr();
2986                 GPRReg gpr = result.gpr();
2987                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
2988                 value.adopt(result);
2989             } else {
2990 #if USE(JSVALUE32_64)
2991                 GPRTemporary realPropertyTag(this);
2992                 propertyTag.adopt(realPropertyTag);
2993                 GPRReg propertyTagGPR = propertyTag.gpr();
2994
2995                 GPRTemporary realValueTag(this);
2996                 valueTag.adopt(realValueTag);
2997                 GPRReg valueTagGPR = valueTag.gpr();
2998 #endif
2999                 SpeculateDoubleOperand valueOp(this, valueUse);
3000                 GPRTemporary result(this);
3001                 FPRReg fpr = valueOp.fpr();
3002                 GPRReg gpr = result.gpr();
3003                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3004                 m_jit.xorPtr(gpr, gpr);
3005                 MacroAssembler::JumpList fixed(m_jit.jump());
3006                 notNaN.link(&m_jit);
3007
3008                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3009                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3010
3011 #if USE(JSVALUE64)
3012                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3013                 boxDouble(fpr, gpr);
3014 #else
3015                 UNUSED_PARAM(property);
3016                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3017                 boxDouble(fpr, valueTagGPR, gpr);
3018 #endif
3019                 slowPathCases.append(m_jit.jump());
3020
3021                 fixed.link(&m_jit);
3022                 value.adopt(result);
3023             }
3024             break;
3025         }
3026             
3027         default:
3028             RELEASE_ASSERT_NOT_REACHED();
3029             break;
3030         }
3031     }
3032     return true;
3033 }
3034
3035 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3036 {
3037     ASSERT(isInt(type));
3038     
3039     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3040     GPRReg storageReg = storage.gpr();
3041     
3042     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3043     
3044     GPRTemporary value;
3045 #if USE(JSVALUE32_64)
3046     GPRTemporary propertyTag;
3047     GPRTemporary valueTag;
3048 #endif
3049
3050     JITCompiler::JumpList slowPathCases;
3051     
3052     bool result = getIntTypedArrayStoreOperand(
3053         value, property,
3054 #if USE(JSVALUE32_64)
3055         propertyTag, valueTag,
3056 #endif
3057         valueUse, slowPathCases, isClamped(type));
3058     if (!result) {
3059         noResult(node);
3060         return;
3061     }
3062
3063     GPRReg valueGPR = value.gpr();
3064 #if USE(JSVALUE32_64)
3065     GPRReg propertyTagGPR = propertyTag.gpr();
3066     GPRReg valueTagGPR = valueTag.gpr();
3067 #endif
3068
3069     ASSERT_UNUSED(valueGPR, valueGPR != property);
3070     ASSERT(valueGPR != base);
3071     ASSERT(valueGPR != storageReg);
3072     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3073
3074     switch (elementSize(type)) {
3075     case 1:
3076         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3077         break;
3078     case 2:
3079         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3080         break;
3081     case 4:
3082         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3083         break;
3084     default:
3085         CRASH();
3086     }
3087
3088     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3089     if (done.isSet())
3090         done.link(&m_jit);
3091
3092     if (!slowPathCases.empty()) {
3093 #if USE(JSVALUE64)
3094         if (node->op() == PutByValDirect) {
3095             addSlowPathGenerator(slowPathCall(
3096                 slowPathCases, this,
3097                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3098                 NoResult, base, property, valueGPR));
3099         } else {
3100             addSlowPathGenerator(slowPathCall(
3101                 slowPathCases, this,
3102                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3103                 NoResult, base, property, valueGPR));
3104         }
3105 #else // not USE(JSVALUE64)
3106         if (node->op() == PutByValDirect) {
3107             addSlowPathGenerator(slowPathCall(
3108                 slowPathCases, this,
3109                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3110                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3111         } else {
3112             addSlowPathGenerator(slowPathCall(
3113                 slowPathCases, this,
3114                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3115                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3116         }
3117 #endif
3118     }
3119     
3120     noResult(node);
3121 }
3122
3123 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3124 {
3125     ASSERT(isFloat(type));
3126     
3127     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3128     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3129     StorageOperand storage(this, m_graph.varArgChild(node, 2));
3130
3131     GPRReg baseReg = base.gpr();
3132     GPRReg propertyReg = property.gpr();
3133     GPRReg storageReg = storage.gpr();
3134
3135     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3136
3137     FPRTemporary result(this);
3138     FPRReg resultReg = result.fpr();
3139     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3140     switch (elementSize(type)) {
3141     case 4:
3142         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3143         m_jit.convertFloatToDouble(resultReg, resultReg);
3144         break;
3145     case 8: {
3146         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3147         break;
3148     }
3149     default:
3150         RELEASE_ASSERT_NOT_REACHED();
3151     }
3152     
3153     doubleResult(resultReg, node);
3154 }
3155
3156 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3157 {
3158     ASSERT(isFloat(type));
3159     
3160     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3161     GPRReg storageReg = storage.gpr();
3162     
3163     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3164     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3165
3166     SpeculateDoubleOperand valueOp(this, valueUse);
3167     FPRTemporary scratch(this);
3168     FPRReg valueFPR = valueOp.fpr();
3169     FPRReg scratchFPR = scratch.fpr();
3170
3171     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3172     
3173     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3174     
3175     switch (elementSize(type)) {
3176     case 4: {
3177         m_jit.moveDouble(valueFPR, scratchFPR);
3178         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3179         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3180         break;
3181     }
3182     case 8:
3183         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3184         break;
3185     default:
3186         RELEASE_ASSERT_NOT_REACHED();
3187     }
3188
3189     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3190     if (done.isSet())
3191         done.link(&m_jit);
3192     noResult(node);
3193 }
3194
3195 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3196 {
3197     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3198     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3199
3200     GPRReg arg1GPR = arg1.gpr();
3201     GPRReg arg2GPR = arg2.gpr();
3202
3203     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3204     speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3205
3206     flushRegisters();
3207     JSValueRegsFlushedCallResult result(this);
3208     JSValueRegs resultRegs = result.regs();
3209     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3210     m_jit.exceptionCheck();
3211
3212     jsValueResult(resultRegs, node);
3213 }
3214
3215 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3216 {
3217     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3218     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3219
3220     GPRReg arg1GPR = arg1.gpr();
3221     GPRReg arg2GPR = arg2.gpr();
3222
3223     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3224     speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3225
3226     flushRegisters();
3227     JSValueRegsFlushedCallResult result(this);
3228     JSValueRegs resultRegs = result.regs();
3229     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3230     m_jit.exceptionCheck();
3231
3232     jsValueResult(resultRegs, node);
3233 }
3234
3235 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3236 {
3237     SpeculateCellOperand arg1(this, child1);
3238     SpeculateCellOperand arg2(this, child2);
3239     JSValueOperand arg3(this, child3);
3240
3241     GPRReg arg1GPR = arg1.gpr();
3242     GPRReg arg2GPR = arg2.gpr();
3243     JSValueRegs arg3Regs = arg3.jsValueRegs();
3244
3245     speculateString(child2, arg2GPR);
3246
3247     flushRegisters();
3248     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3249     m_jit.exceptionCheck();
3250
3251     noResult(node);
3252 }
3253
3254 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3255 {
3256     SpeculateCellOperand arg1(this, child1);
3257     SpeculateCellOperand arg2(this, child2);
3258     JSValueOperand arg3(this, child3);
3259
3260     GPRReg arg1GPR = arg1.gpr();
3261     GPRReg arg2GPR = arg2.gpr();
3262     JSValueRegs arg3Regs = arg3.jsValueRegs();
3263
3264     speculateSymbol(child2, arg2GPR);
3265
3266     flushRegisters();
3267     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3268     m_jit.exceptionCheck();
3269
3270     noResult(node);
3271 }
3272
3273 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg, GPRReg scratch3Reg)
3274 {
3275     // Check that prototype is an object.
3276     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
3277     
3278     // Initialize scratchReg with the value being checked.
3279     m_jit.move(valueReg, scratchReg);
3280     
3281     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
3282     MacroAssembler::Label loop(&m_jit);
3283     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
3284         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
3285     m_jit.emitLoadStructure(*m_jit.vm(), scratchReg, scratch3Reg, scratch2Reg);
3286 #if USE(JSVALUE64)
3287     m_jit.load64(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset()), scratch3Reg);
3288     auto hasMonoProto = m_jit.branchTest64(JITCompiler::NonZero, scratch3Reg);
3289     m_jit.load64(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset)), scratch3Reg);
3290     hasMonoProto.link(&m_jit);
3291     m_jit.move(scratch3Reg, scratchReg);
3292 #else
3293     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + TagOffset), scratch2Reg);
3294     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + PayloadOffset), scratch3Reg);
3295     auto hasMonoProto = m_jit.branch32(CCallHelpers::NotEqual, scratch2Reg, TrustedImm32(JSValue::EmptyValueTag));
3296     m_jit.load32(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), scratch3Reg);
3297     hasMonoProto.link(&m_jit);
3298     m_jit.move(scratch3Reg, scratchReg);
3299 #endif
3300
3301     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
3302 #if USE(JSVALUE64)
3303     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
3304 #else
3305     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
3306 #endif
3307     
3308     // No match - result is false.
3309 #if USE(JSVALUE64)
3310     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
3311 #else
3312     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
3313 #endif
3314     MacroAssembler::JumpList doneJumps; 
3315     doneJumps.append(m_jit.jump());
3316
3317     performDefaultHasInstance.link(&m_jit);
3318     silentSpillAllRegisters(scratchReg);
3319     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg); 
3320     silentFillAllRegisters();
3321     m_jit.exceptionCheck();
3322 #if USE(JSVALUE64)
3323     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
3324 #endif
3325     doneJumps.append(m_jit.jump());
3326     
3327     isInstance.link(&m_jit);
3328 #if USE(JSVALUE64)
3329     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), scratchReg);
3330 #else
3331     m_jit.move(MacroAssembler::TrustedImm32(1), scratchReg);
3332 #endif
3333     
3334     doneJumps.link(&m_jit);
3335 }
3336
3337 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3338 {
3339     SpeculateCellOperand base(this, node->child1());
3340
3341     GPRReg baseGPR = base.gpr();
3342
3343     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3344
3345     noResult(node);
3346 }
3347
3348 void SpeculativeJIT::compileParseInt(Node* node)
3349 {
3350     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3351     if (node->child2()) {
3352         SpeculateInt32Operand radix(this, node->child2());
3353         GPRReg radixGPR = radix.gpr();
3354         if (node->child1().useKind() == UntypedUse) {
3355             JSValueOperand value(this, node->child1());
3356             JSValueRegs valueRegs = value.jsValueRegs();
3357
3358             flushRegisters();
3359             JSValueRegsFlushedCallResult result(this);
3360             JSValueRegs resultRegs = result.regs();
3361             callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3362             m_jit.exceptionCheck();
3363             jsValueResult(resultRegs, node);
3364             return;
3365         }
3366
3367         SpeculateCellOperand value(this, node->child1());
3368         GPRReg valueGPR = value.gpr();
3369         speculateString(node->child1(), valueGPR);
3370
3371         flushRegisters();
3372         JSValueRegsFlushedCallResult result(this);
3373         JSValueRegs resultRegs = result.regs();
3374         callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3375         m_jit.exceptionCheck();
3376         jsValueResult(resultRegs, node);
3377         return;
3378     }
3379
3380     if (node->child1().useKind() == UntypedUse) {
3381         JSValueOperand value(this, node->child1());
3382         JSValueRegs valueRegs = value.jsValueRegs();
3383
3384         flushRegisters();
3385         JSValueRegsFlushedCallResult result(this);
3386         JSValueRegs resultRegs = result.regs();
3387         callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3388         m_jit.exceptionCheck();
3389         jsValueResult(resultRegs, node);
3390         return;
3391     }
3392
3393     SpeculateCellOperand value(this, node->child1());
3394     GPRReg valueGPR = value.gpr();
3395     speculateString(node->child1(), valueGPR);
3396
3397     flushRegisters();
3398     JSValueRegsFlushedCallResult result(this);
3399     JSValueRegs resultRegs = result.regs();
3400     callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3401     m_jit.exceptionCheck();
3402     jsValueResult(resultRegs, node);
3403 }
3404
3405 void SpeculativeJIT::compileInstanceOf(Node* node)
3406 {
3407     if (node->child1().useKind() == UntypedUse) {
3408         // It might not be a cell. Speculate less aggressively.
3409         // Or: it might only be used once (i.e. by us), so we get zero benefit
3410         // from speculating any more aggressively than we absolutely need to.
3411         
3412         JSValueOperand value(this, node->child1());
3413         SpeculateCellOperand prototype(this, node->child2());
3414         GPRTemporary scratch(this);
3415         GPRTemporary scratch2(this);
3416         GPRTemporary scratch3(this);
3417         
3418         GPRReg prototypeReg = prototype.gpr();
3419         GPRReg scratchReg = scratch.gpr();
3420         GPRReg scratch2Reg = scratch2.gpr();
3421         GPRReg scratch3Reg = scratch3.gpr();
3422         
3423         MacroAssembler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs());
3424         GPRReg valueReg = value.jsValueRegs().payloadGPR();
3425         moveFalseTo(scratchReg);
3426
3427         MacroAssembler::Jump done = m_jit.jump();
3428         
3429         isCell.link(&m_jit);
3430         
3431         compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3432         
3433         done.link(&m_jit);
3434
3435         blessedBooleanResult(scratchReg, node);
3436         return;
3437     }
3438     
3439     SpeculateCellOperand value(this, node->child1());
3440     SpeculateCellOperand prototype(this, node->child2());
3441     
3442     GPRTemporary scratch(this);
3443     GPRTemporary scratch2(this);
3444     GPRTemporary scratch3(this);
3445     
3446     GPRReg valueReg = value.gpr();
3447     GPRReg prototypeReg = prototype.gpr();
3448     GPRReg scratchReg = scratch.gpr();
3449     GPRReg scratch2Reg = scratch2.gpr();
3450     GPRReg scratch3Reg = scratch3.gpr();
3451     
3452     compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3453
3454     blessedBooleanResult(scratchReg, node);
3455 }
3456
3457 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3458 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3459 {
3460     Edge& leftChild = node->child1();
3461     Edge& rightChild = node->child2();
3462
3463     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3464         JSValueOperand left(this, leftChild);
3465         JSValueOperand right(this, rightChild);
3466         JSValueRegs leftRegs = left.jsValueRegs();
3467         JSValueRegs rightRegs = right.jsValueRegs();
3468
3469         flushRegisters();
3470         JSValueRegsFlushedCallResult result(this);
3471         JSValueRegs resultRegs = result.regs();
3472         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3473         m_jit.exceptionCheck();
3474
3475         jsValueResult(resultRegs, node);
3476         return;
3477     }
3478
3479     std::optional<JSValueOperand> left;
3480     std::optional<JSValueOperand> right;
3481
3482     JSValueRegs leftRegs;
3483     JSValueRegs rightRegs;
3484
3485 #if USE(JSVALUE64)
3486     GPRTemporary result(this);
3487     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3488     GPRTemporary scratch(this);
3489     GPRReg scratchGPR = scratch.gpr();
3490 #else
3491     GPRTemporary resultTag(this);
3492     GPRTemporary resultPayload(this);
3493     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3494     GPRReg scratchGPR = resultTag.gpr();
3495 #endif
3496
3497     SnippetOperand leftOperand;
3498     SnippetOperand rightOperand;
3499
3500     // The snippet generator does not support both operands being constant. If the left
3501     // operand is already const, we'll ignore the right operand's constness.
3502     if (leftChild->isInt32Constant())
3503         leftOperand.setConstInt32(leftChild->asInt32());
3504     else if (rightChild->isInt32Constant())
3505         rightOperand.setConstInt32(rightChild->asInt32());
3506
3507     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3508
3509     if (!leftOperand.isConst()) {
3510         left.emplace(this, leftChild);
3511         leftRegs = left->jsValueRegs();
3512     }
3513     if (!rightOperand.isConst()) {
3514         right.emplace(this, rightChild);
3515         rightRegs = right->jsValueRegs();
3516     }
3517
3518     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3519     gen.generateFastPath(m_jit);
3520
3521     ASSERT(gen.didEmitFastPath());
3522     gen.endJumpList().append(m_jit.jump());
3523
3524     gen.slowPathJumpList().link(&m_jit);
3525     silentSpillAllRegisters(resultRegs);
3526
3527     if (leftOperand.isConst()) {
3528         leftRegs = resultRegs;
3529         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3530     } else if (rightOperand.isConst()) {
3531         rightRegs = resultRegs;
3532         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3533     }
3534
3535     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3536
3537     silentFillAllRegisters();
3538     m_jit.exceptionCheck();
3539
3540     gen.endJumpList().link(&m_jit);
3541     jsValueResult(resultRegs, node);
3542 }
3543
3544 void SpeculativeJIT::compileBitwiseOp(Node* node)
3545 {
3546     NodeType op = node->op();
3547     Edge& leftChild = node->child1();
3548     Edge& rightChild = node->child2();
3549
3550     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3551         switch (op) {
3552         case BitAnd:
3553             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3554             return;
3555         case BitOr:
3556             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3557             return;
3558         case BitXor:
3559             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3560             return;
3561         default:
3562             RELEASE_ASSERT_NOT_REACHED();
3563         }
3564     }
3565
3566     if (leftChild->isInt32Constant()) {
3567         SpeculateInt32Operand op2(this, rightChild);
3568         GPRTemporary result(this, Reuse, op2);
3569
3570         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3571
3572         int32Result(result.gpr(), node);
3573
3574     } else if (rightChild->isInt32Constant()) {
3575         SpeculateInt32Operand op1(this, leftChild);
3576         GPRTemporary result(this, Reuse, op1);
3577
3578         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3579
3580         int32Result(result.gpr(), node);
3581
3582     } else {
3583         SpeculateInt32Operand op1(this, leftChild);
3584         SpeculateInt32Operand op2(this, rightChild);
3585         GPRTemporary result(this, Reuse, op1, op2);
3586         
3587         GPRReg reg1 = op1.gpr();
3588         GPRReg reg2 = op2.gpr();
3589         bitOp(op, reg1, reg2, result.gpr());
3590         
3591         int32Result(result.gpr(), node);
3592     }
3593 }
3594
3595 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3596 {
3597     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3598         ? operationValueBitRShift : operationValueBitURShift;
3599     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3600         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3601
3602     Edge& leftChild = node->child1();
3603     Edge& rightChild = node->child2();
3604
3605     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3606         JSValueOperand left(this, leftChild);
3607         JSValueOperand right(this, rightChild);
3608         JSValueRegs leftRegs = left.jsValueRegs();
3609         JSValueRegs rightRegs = right.jsValueRegs();
3610
3611         flushRegisters();
3612         JSValueRegsFlushedCallResult result(this);
3613         JSValueRegs resultRegs = result.regs();
3614         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3615         m_jit.exceptionCheck();
3616
3617         jsValueResult(resultRegs, node);
3618         return;
3619     }
3620
3621     std::optional<JSValueOperand> left;
3622     std::optional<JSValueOperand> right;
3623
3624     JSValueRegs leftRegs;
3625     JSValueRegs rightRegs;
3626
3627     FPRTemporary leftNumber(this);
3628     FPRReg leftFPR = leftNumber.fpr();
3629
3630 #if USE(JSVALUE64)
3631     GPRTemporary result(this);
3632     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3633     GPRTemporary scratch(this);
3634     GPRReg scratchGPR = scratch.gpr();
3635     FPRReg scratchFPR = InvalidFPRReg;
3636 #else
3637     GPRTemporary resultTag(this);
3638     GPRTemporary resultPayload(this);
3639     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3640     GPRReg scratchGPR = resultTag.gpr();
3641     FPRTemporary fprScratch(this);
3642     FPRReg scratchFPR = fprScratch.fpr();
3643 #endif
3644
3645     SnippetOperand leftOperand;
3646     SnippetOperand rightOperand;
3647
3648     // The snippet generator does not support both operands being constant. If the left
3649     // operand is already const, we'll ignore the right operand's constness.
3650     if (leftChild->isInt32Constant())
3651         leftOperand.setConstInt32(leftChild->asInt32());
3652     else if (rightChild->isInt32Constant())
3653         rightOperand.setConstInt32(rightChild->asInt32());
3654
3655     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3656
3657     if (!leftOperand.isConst()) {
3658         left.emplace(this, leftChild);
3659         leftRegs = left->jsValueRegs();
3660     }
3661     if (!rightOperand.isConst()) {
3662         right.emplace(this, rightChild);
3663         rightRegs = right->jsValueRegs();
3664     }
3665
3666     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3667         leftFPR, scratchGPR, scratchFPR, shiftType);
3668     gen.generateFastPath(m_jit);
3669
3670     ASSERT(gen.didEmitFastPath());
3671     gen.endJumpList().append(m_jit.jump());
3672
3673     gen.slowPathJumpList().link(&m_jit);
3674     silentSpillAllRegisters(resultRegs);
3675
3676     if (leftOperand.isConst()) {
3677         leftRegs = resultRegs;
3678         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3679     } else if (rightOperand.isConst()) {
3680         rightRegs = resultRegs;
3681         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3682     }
3683
3684     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3685
3686     silentFillAllRegisters();
3687     m_jit.exceptionCheck();
3688
3689     gen.endJumpList().link(&m_jit);
3690     jsValueResult(resultRegs, node);
3691     return;
3692 }
3693
3694 void SpeculativeJIT::compileShiftOp(Node* node)
3695 {
3696     NodeType op = node->op();
3697     Edge& leftChild = node->child1();
3698     Edge& rightChild = node->child2();
3699
3700     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3701         switch (op) {
3702         case BitLShift:
3703             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3704             return;
3705         case BitRShift:
3706         case BitURShift:
3707             emitUntypedRightShiftBitOp(node);
3708             return;
3709         default:
3710             RELEASE_ASSERT_NOT_REACHED();
3711         }
3712     }
3713
3714     if (rightChild->isInt32Constant()) {
3715         SpeculateInt32Operand op1(this, leftChild);
3716         GPRTemporary result(this, Reuse, op1);
3717
3718         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3719
3720         int32Result(result.gpr(), node);
3721     } else {
3722         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3723         SpeculateInt32Operand op1(this, leftChild);
3724         SpeculateInt32Operand op2(this, rightChild);
3725         GPRTemporary result(this, Reuse, op1);
3726
3727         GPRReg reg1 = op1.gpr();
3728         GPRReg reg2 = op2.gpr();
3729         shiftOp(op, reg1, reg2, result.gpr());
3730
3731         int32Result(result.gpr(), node);
3732     }
3733 }
3734
3735 void SpeculativeJIT::compileValueAdd(Node* node)
3736 {
3737     Edge& leftChild = node->child1();
3738     Edge& rightChild = node->child2();
3739
3740     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3741         JSValueOperand left(this, leftChild);
3742         JSValueOperand right(this, rightChild);
3743         JSValueRegs leftRegs = left.jsValueRegs();
3744         JSValueRegs rightRegs = right.jsValueRegs();
3745
3746         flushRegisters();
3747         JSValueRegsFlushedCallResult result(this);
3748         JSValueRegs resultRegs = result.regs();
3749         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3750         m_jit.exceptionCheck();
3751     
3752         jsValueResult(resultRegs, node);
3753         return;
3754     }
3755
3756 #if USE(JSVALUE64)
3757     bool needsScratchGPRReg = true;
3758     bool needsScratchFPRReg = false;
3759 #else
3760     bool needsScratchGPRReg = true;
3761     bool needsScratchFPRReg = true;
3762 #endif
3763
3764     CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3765     ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(node->origin.semantic.bytecodeIndex);
3766     Instruction* instruction = &baselineCodeBlock->instructions()[node->origin.semantic.bytecodeIndex];
3767     JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile, instruction);
3768     auto repatchingFunction = operationValueAddOptimize;
3769     auto nonRepatchingFunction = operationValueAdd;
3770     
3771     compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3772 }
3773
3774 template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3775 void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3776 {
3777     Edge& leftChild = node->child1();
3778     Edge& rightChild = node->child2();
3779
3780     std::optional<JSValueOperand> left;
3781     std::optional<JSValueOperand> right;
3782
3783     JSValueRegs leftRegs;
3784     JSValueRegs rightRegs;
3785
3786     FPRTemporary leftNumber(this);
3787     FPRTemporary rightNumber(this);
3788     FPRReg leftFPR = leftNumber.fpr();
3789     FPRReg rightFPR = rightNumber.fpr();
3790
3791     GPRReg scratchGPR = InvalidGPRReg;
3792     FPRReg scratchFPR = InvalidFPRReg;
3793
3794     std::optional<FPRTemporary> fprScratch;
3795     if (needsScratchFPRReg) {
3796         fprScratch.emplace(this);
3797         scratchFPR = fprScratch->fpr();
3798     }
3799
3800 #if USE(JSVALUE64)
3801     std::optional<GPRTemporary> gprScratch;
3802     if (needsScratchGPRReg) {
3803         gprScratch.emplace(this);
3804         scratchGPR = gprScratch->gpr();
3805     }
3806     GPRTemporary result(this);
3807     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3808 #else
3809     GPRTemporary resultTag(this);
3810     GPRTemporary resultPayload(this);
3811     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3812     if (needsScratchGPRReg)
3813         scratchGPR = resultRegs.tagGPR();
3814 #endif
3815
3816     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3817     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3818
3819     // The snippet generator does not support both operands being constant. If the left
3820     // operand is already const, we'll ignore the right operand's constness.
3821     if (leftChild->isInt32Constant())
3822         leftOperand.setConstInt32(leftChild->asInt32());
3823     else if (rightChild->isInt32Constant())
3824         rightOperand.setConstInt32(rightChild->asInt32());
3825
3826     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3827     ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
3828
3829     if (!Generator::isLeftOperandValidConstant(leftOperand)) {
3830         left.emplace(this, leftChild);
3831         leftRegs = left->jsValueRegs();
3832     }
3833     if (!Generator::isRightOperandValidConstant(rightOperand)) {
3834         right.emplace(this, rightChild);
3835         rightRegs = right->jsValueRegs();
3836     }
3837
3838 #if ENABLE(MATH_IC_STATS)
3839     auto inlineStart = m_jit.label();
3840 #endif
3841
3842     Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
3843     mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
3844
3845     bool shouldEmitProfiling = false;
3846     bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
3847
3848     if (generatedInline) {
3849         ASSERT(!addICGenerationState->slowPathJumps.empty());
3850
3851         Vector<SilentRegisterSavePlan> savePlans;
3852         silentSpillAllRegistersImpl(false, savePlans, resultRegs);
3853
3854         auto done = m_jit.label();
3855
3856         addSlowPathGenerator([=, savePlans = WTFMove(savePlans)] () {
3857             addICGenerationState->slowPathJumps.link(&m_jit);
3858             addICGenerationState->slowPathStart = m_jit.label();
3859 #if ENABLE(MATH_IC_STATS)
3860             auto slowPathStart = m_jit.label();
3861 #endif
3862
3863             silentSpill(savePlans);
3864
3865             auto innerLeftRegs = leftRegs;
3866             auto innerRightRegs = rightRegs;
3867             if (Generator::isLeftOperandValidConstant(leftOperand)) {
3868                 innerLeftRegs = resultRegs;
3869                 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
3870             } else if (Generator::isRightOperandValidConstant(rightOperand)) {
3871                 innerRightRegs = resultRegs;
3872                 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
3873             }
3874
3875             if (addICGenerationState->shouldSlowPathRepatch)
3876                 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
3877             else
3878                 addICGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, innerLeftRegs, innerRightRegs);
3879
3880             silentFill(savePlans);
3881             m_jit.exceptionCheck();
3882             m_jit.jump().linkTo(done, &m_jit);
3883
3884             m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
3885                 mathIC->finalizeInlineCode(*addICGenerationState, linkBuffer);
3886             });
3887
3888 #if ENABLE(MATH_IC_STATS)
3889             auto slowPathEnd = m_jit.label();
3890             m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
3891                 size_t size = static_cast<char*>(linkBuffer.locationOf(slowPathEnd).executableAddress()) - static_cast<char*>(linkBuffer.locationOf(slowPathStart).executableAddress());
3892                 mathIC->m_generatedCodeSize += size;
3893             });