[Re-landing] DFG::SpeculativeJIT::pickCanTrample() is wrongly ignoring result registers.
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCapabilities.h"
37 #include "DFGDOMJITPatchpointParams.h"
38 #include "DFGMayExit.h"
39 #include "DFGOSRExitFuzz.h"
40 #include "DFGSaneStringGetByValSlowPathGenerator.h"
41 #include "DFGSlowPathGenerator.h"
42 #include "DOMJITPatchpoint.h"
43 #include "DirectArguments.h"
44 #include "JITAddGenerator.h"
45 #include "JITBitAndGenerator.h"
46 #include "JITBitOrGenerator.h"
47 #include "JITBitXorGenerator.h"
48 #include "JITDivGenerator.h"
49 #include "JITLeftShiftGenerator.h"
50 #include "JITMulGenerator.h"
51 #include "JITRightShiftGenerator.h"
52 #include "JITSubGenerator.h"
53 #include "JSAsyncFunction.h"
54 #include "JSCInlines.h"
55 #include "JSEnvironmentRecord.h"
56 #include "JSFixedArray.h"
57 #include "JSGeneratorFunction.h"
58 #include "JSLexicalEnvironment.h"
59 #include "LinkBuffer.h"
60 #include "RegExpConstructor.h"
61 #include "ScopedArguments.h"
62 #include "ScratchRegisterAllocator.h"
63 #include <wtf/BitVector.h>
64 #include <wtf/Box.h>
65 #include <wtf/MathExtras.h>
66
67 namespace JSC { namespace DFG {
68
69 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
70     : m_compileOkay(true)
71     , m_jit(jit)
72     , m_currentNode(0)
73     , m_lastGeneratedNode(LastNodeType)
74     , m_indexInBlock(0)
75     , m_generationInfo(m_jit.graph().frameRegisterCount())
76     , m_state(m_jit.graph())
77     , m_interpreter(m_jit.graph(), m_state)
78     , m_stream(&jit.jitCode()->variableEventStream)
79     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
80 {
81 }
82
83 SpeculativeJIT::~SpeculativeJIT()
84 {
85 }
86
87 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
88 {
89     IndexingType indexingType = structure->indexingType();
90     bool hasIndexingHeader = hasIndexedProperties(indexingType);
91
92     unsigned inlineCapacity = structure->inlineCapacity();
93     unsigned outOfLineCapacity = structure->outOfLineCapacity();
94     
95     GPRTemporary scratch(this);
96     GPRTemporary scratch2(this);
97     GPRReg scratchGPR = scratch.gpr();
98     GPRReg scratch2GPR = scratch2.gpr();
99
100     ASSERT(vectorLength >= numElements);
101     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
102     
103     JITCompiler::JumpList slowCases;
104
105     size_t size = 0;
106     if (hasIndexingHeader)
107         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
108     size += outOfLineCapacity * sizeof(JSValue);
109
110     m_jit.move(TrustedImmPtr(0), storageGPR);
111     
112     if (size) {
113         if (MarkedAllocator* allocator = m_jit.vm()->auxiliarySpace.allocatorFor(size)) {
114             m_jit.move(TrustedImmPtr(allocator), scratchGPR);
115             m_jit.emitAllocate(storageGPR, allocator, scratchGPR, scratch2GPR, slowCases);
116             
117             m_jit.addPtr(
118                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
119                 storageGPR);
120             
121             if (hasIndexingHeader)
122                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
123         } else
124             slowCases.append(m_jit.jump());
125     }
126
127     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
128     MarkedAllocator* allocatorPtr = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorFor(allocationSize);
129     if (allocatorPtr) {
130         m_jit.move(TrustedImmPtr(allocatorPtr), scratchGPR);
131         emitAllocateJSObject(resultGPR, allocatorPtr, scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
132         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
133     } else
134         slowCases.append(m_jit.jump());
135
136     // I want a slow path that also loads out the storage pointer, and that's
137     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
138     // of work for a very small piece of functionality. :-/
139     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
140         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
141         structure, vectorLength));
142
143     if (numElements < vectorLength) {
144 #if USE(JSVALUE64)
145         if (hasDouble(structure->indexingType()))
146             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
147         else
148             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
149         for (unsigned i = numElements; i < vectorLength; ++i)
150             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
151 #else
152         EncodedValueDescriptor value;
153         if (hasDouble(structure->indexingType()))
154             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
155         else
156             value.asInt64 = JSValue::encode(JSValue());
157         for (unsigned i = numElements; i < vectorLength; ++i) {
158             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
159             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
160         }
161 #endif
162     }
163     
164     if (hasIndexingHeader)
165         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
166     
167     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
168     
169     m_jit.mutatorFence(*m_jit.vm());
170 }
171
172 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
173 {
174     if (inlineCallFrame && !inlineCallFrame->isVarargs())
175         m_jit.move(TrustedImm32(inlineCallFrame->arguments.size() - !includeThis), lengthGPR);
176     else {
177         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
178         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
179         if (!includeThis)
180             m_jit.sub32(TrustedImm32(1), lengthGPR);
181     }
182 }
183
184 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
185 {
186     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
187 }
188
189 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
190 {
191     if (origin.inlineCallFrame) {
192         if (origin.inlineCallFrame->isClosureCall) {
193             m_jit.loadPtr(
194                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
195                 calleeGPR);
196         } else {
197             m_jit.move(
198                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
199                 calleeGPR);
200         }
201     } else
202         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
203 }
204
205 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
206 {
207     m_jit.addPtr(
208         TrustedImm32(
209             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
210         GPRInfo::callFrameRegister, startGPR);
211 }
212
213 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
214 {
215     if (!Options::useOSRExitFuzz()
216         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
217         || !doOSRExitFuzzing())
218         return MacroAssembler::Jump();
219     
220     MacroAssembler::Jump result;
221     
222     m_jit.pushToSave(GPRInfo::regT0);
223     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
224     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
225     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
226     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
227     unsigned at = Options::fireOSRExitFuzzAt();
228     if (at || atOrAfter) {
229         unsigned threshold;
230         MacroAssembler::RelationalCondition condition;
231         if (atOrAfter) {
232             threshold = atOrAfter;
233             condition = MacroAssembler::Below;
234         } else {
235             threshold = at;
236             condition = MacroAssembler::NotEqual;
237         }
238         MacroAssembler::Jump ok = m_jit.branch32(
239             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
240         m_jit.popToRestore(GPRInfo::regT0);
241         result = m_jit.jump();
242         ok.link(&m_jit);
243     }
244     m_jit.popToRestore(GPRInfo::regT0);
245     
246     return result;
247 }
248
249 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
250 {
251     if (!m_compileOkay)
252         return;
253     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
254     if (fuzzJump.isSet()) {
255         JITCompiler::JumpList jumpsToFail;
256         jumpsToFail.append(fuzzJump);
257         jumpsToFail.append(jumpToFail);
258         m_jit.appendExitInfo(jumpsToFail);
259     } else
260         m_jit.appendExitInfo(jumpToFail);
261     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
262 }
263
264 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
265 {
266     if (!m_compileOkay)
267         return;
268     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
269     if (fuzzJump.isSet()) {
270         JITCompiler::JumpList myJumpsToFail;
271         myJumpsToFail.append(jumpsToFail);
272         myJumpsToFail.append(fuzzJump);
273         m_jit.appendExitInfo(myJumpsToFail);
274     } else
275         m_jit.appendExitInfo(jumpsToFail);
276     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
277 }
278
279 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
280 {
281     if (!m_compileOkay)
282         return OSRExitJumpPlaceholder();
283     unsigned index = m_jit.jitCode()->osrExit.size();
284     m_jit.appendExitInfo();
285     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
286     return OSRExitJumpPlaceholder(index);
287 }
288
289 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
290 {
291     return speculationCheck(kind, jsValueSource, nodeUse.node());
292 }
293
294 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
295 {
296     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
297 }
298
299 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
300 {
301     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
302 }
303
304 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
305 {
306     if (!m_compileOkay)
307         return;
308     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
309     m_jit.appendExitInfo(jumpToFail);
310     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
311 }
312
313 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
314 {
315     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
316 }
317
318 void SpeculativeJIT::emitInvalidationPoint(Node* node)
319 {
320     if (!m_compileOkay)
321         return;
322     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
323     m_jit.jitCode()->appendOSRExit(OSRExit(
324         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
325         this, m_stream->size()));
326     info.m_replacementSource = m_jit.watchpointLabel();
327     ASSERT(info.m_replacementSource.isSet());
328     noResult(node);
329 }
330
331 void SpeculativeJIT::unreachable(Node* node)
332 {
333     m_compileOkay = false;
334     m_jit.abortWithReason(DFGUnreachableNode, node->op());
335 }
336
337 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
338 {
339     if (!m_compileOkay)
340         return;
341     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
342     m_compileOkay = false;
343     if (verboseCompilationEnabled())
344         dataLog("Bailing compilation.\n");
345 }
346
347 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
348 {
349     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
350 }
351
352 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
353 {
354     ASSERT(needsTypeCheck(edge, typesPassedThrough));
355     m_interpreter.filter(edge, typesPassedThrough);
356     speculationCheck(exitKind, source, edge.node(), jumpToFail);
357 }
358
359 RegisterSet SpeculativeJIT::usedRegisters()
360 {
361     RegisterSet result;
362     
363     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
364         GPRReg gpr = GPRInfo::toRegister(i);
365         if (m_gprs.isInUse(gpr))
366             result.set(gpr);
367     }
368     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
369         FPRReg fpr = FPRInfo::toRegister(i);
370         if (m_fprs.isInUse(fpr))
371             result.set(fpr);
372     }
373     
374     result.merge(RegisterSet::stubUnavailableRegisters());
375     
376     return result;
377 }
378
379 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
380 {
381     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
382 }
383
384 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
385 {
386     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
387 }
388
389 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
390 {
391     for (auto& slowPathGenerator : m_slowPathGenerators) {
392         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
393         slowPathGenerator->generate(this);
394     }
395     for (auto& slowPathLambda : m_slowPathLambdas) {
396         Node* currentNode = slowPathLambda.currentNode;
397         m_currentNode = currentNode;
398         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
399         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
400         slowPathLambda.generator();
401         m_outOfLineStreamIndex = std::nullopt;
402     }
403 }
404
405 void SpeculativeJIT::clearGenerationInfo()
406 {
407     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
408         m_generationInfo[i] = GenerationInfo();
409     m_gprs = RegisterBank<GPRInfo>();
410     m_fprs = RegisterBank<FPRInfo>();
411 }
412
413 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
414 {
415     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
416     Node* node = info.node();
417     DataFormat registerFormat = info.registerFormat();
418     ASSERT(registerFormat != DataFormatNone);
419     ASSERT(registerFormat != DataFormatDouble);
420         
421     SilentSpillAction spillAction;
422     SilentFillAction fillAction;
423         
424     if (!info.needsSpill())
425         spillAction = DoNothingForSpill;
426     else {
427 #if USE(JSVALUE64)
428         ASSERT(info.gpr() == source);
429         if (registerFormat == DataFormatInt32)
430             spillAction = Store32Payload;
431         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
432             spillAction = StorePtr;
433         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
434             spillAction = Store64;
435         else {
436             ASSERT(registerFormat & DataFormatJS);
437             spillAction = Store64;
438         }
439 #elif USE(JSVALUE32_64)
440         if (registerFormat & DataFormatJS) {
441             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
442             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
443         } else {
444             ASSERT(info.gpr() == source);
445             spillAction = Store32Payload;
446         }
447 #endif
448     }
449         
450     if (registerFormat == DataFormatInt32) {
451         ASSERT(info.gpr() == source);
452         ASSERT(isJSInt32(info.registerFormat()));
453         if (node->hasConstant()) {
454             ASSERT(node->isInt32Constant());
455             fillAction = SetInt32Constant;
456         } else
457             fillAction = Load32Payload;
458     } else if (registerFormat == DataFormatBoolean) {
459 #if USE(JSVALUE64)
460         RELEASE_ASSERT_NOT_REACHED();
461 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
462         fillAction = DoNothingForFill;
463 #endif
464 #elif USE(JSVALUE32_64)
465         ASSERT(info.gpr() == source);
466         if (node->hasConstant()) {
467             ASSERT(node->isBooleanConstant());
468             fillAction = SetBooleanConstant;
469         } else
470             fillAction = Load32Payload;
471 #endif
472     } else if (registerFormat == DataFormatCell) {
473         ASSERT(info.gpr() == source);
474         if (node->hasConstant()) {
475             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
476             node->asCell(); // To get the assertion.
477             fillAction = SetCellConstant;
478         } else {
479 #if USE(JSVALUE64)
480             fillAction = LoadPtr;
481 #else
482             fillAction = Load32Payload;
483 #endif
484         }
485     } else if (registerFormat == DataFormatStorage) {
486         ASSERT(info.gpr() == source);
487         fillAction = LoadPtr;
488     } else if (registerFormat == DataFormatInt52) {
489         if (node->hasConstant())
490             fillAction = SetInt52Constant;
491         else if (info.spillFormat() == DataFormatInt52)
492             fillAction = Load64;
493         else if (info.spillFormat() == DataFormatStrictInt52)
494             fillAction = Load64ShiftInt52Left;
495         else if (info.spillFormat() == DataFormatNone)
496             fillAction = Load64;
497         else {
498             RELEASE_ASSERT_NOT_REACHED();
499 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
500             fillAction = Load64; // Make GCC happy.
501 #endif
502         }
503     } else if (registerFormat == DataFormatStrictInt52) {
504         if (node->hasConstant())
505             fillAction = SetStrictInt52Constant;
506         else if (info.spillFormat() == DataFormatInt52)
507             fillAction = Load64ShiftInt52Right;
508         else if (info.spillFormat() == DataFormatStrictInt52)
509             fillAction = Load64;
510         else if (info.spillFormat() == DataFormatNone)
511             fillAction = Load64;
512         else {
513             RELEASE_ASSERT_NOT_REACHED();
514 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
515             fillAction = Load64; // Make GCC happy.
516 #endif
517         }
518     } else {
519         ASSERT(registerFormat & DataFormatJS);
520 #if USE(JSVALUE64)
521         ASSERT(info.gpr() == source);
522         if (node->hasConstant()) {
523             if (node->isCellConstant())
524                 fillAction = SetTrustedJSConstant;
525             else
526                 fillAction = SetJSConstant;
527         } else if (info.spillFormat() == DataFormatInt32) {
528             ASSERT(registerFormat == DataFormatJSInt32);
529             fillAction = Load32PayloadBoxInt;
530         } else
531             fillAction = Load64;
532 #else
533         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
534         if (node->hasConstant())
535             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
536         else if (info.payloadGPR() == source)
537             fillAction = Load32Payload;
538         else { // Fill the Tag
539             switch (info.spillFormat()) {
540             case DataFormatInt32:
541                 ASSERT(registerFormat == DataFormatJSInt32);
542                 fillAction = SetInt32Tag;
543                 break;
544             case DataFormatCell:
545                 ASSERT(registerFormat == DataFormatJSCell);
546                 fillAction = SetCellTag;
547                 break;
548             case DataFormatBoolean:
549                 ASSERT(registerFormat == DataFormatJSBoolean);
550                 fillAction = SetBooleanTag;
551                 break;
552             default:
553                 fillAction = Load32Tag;
554                 break;
555             }
556         }
557 #endif
558     }
559         
560     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
561 }
562     
563 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
564 {
565     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
566     Node* node = info.node();
567     ASSERT(info.registerFormat() == DataFormatDouble);
568
569     SilentSpillAction spillAction;
570     SilentFillAction fillAction;
571         
572     if (!info.needsSpill())
573         spillAction = DoNothingForSpill;
574     else {
575         ASSERT(!node->hasConstant());
576         ASSERT(info.spillFormat() == DataFormatNone);
577         ASSERT(info.fpr() == source);
578         spillAction = StoreDouble;
579     }
580         
581 #if USE(JSVALUE64)
582     if (node->hasConstant()) {
583         node->asNumber(); // To get the assertion.
584         fillAction = SetDoubleConstant;
585     } else {
586         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
587         fillAction = LoadDouble;
588     }
589 #elif USE(JSVALUE32_64)
590     ASSERT(info.registerFormat() == DataFormatDouble);
591     if (node->hasConstant()) {
592         node->asNumber(); // To get the assertion.
593         fillAction = SetDoubleConstant;
594     } else
595         fillAction = LoadDouble;
596 #endif
597
598     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
599 }
600     
601 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
602 {
603     switch (plan.spillAction()) {
604     case DoNothingForSpill:
605         break;
606     case Store32Tag:
607         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
608         break;
609     case Store32Payload:
610         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
611         break;
612     case StorePtr:
613         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
614         break;
615 #if USE(JSVALUE64)
616     case Store64:
617         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
618         break;
619 #endif
620     case StoreDouble:
621         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
622         break;
623     default:
624         RELEASE_ASSERT_NOT_REACHED();
625     }
626 }
627     
628 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
629 {
630     switch (plan.fillAction()) {
631     case DoNothingForFill:
632         break;
633     case SetInt32Constant:
634         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
635         break;
636 #if USE(JSVALUE64)
637     case SetInt52Constant:
638         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
639         break;
640     case SetStrictInt52Constant:
641         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
642         break;
643 #endif // USE(JSVALUE64)
644     case SetBooleanConstant:
645         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
646         break;
647     case SetCellConstant:
648         ASSERT(plan.node()->constant()->value().isCell());
649         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
650         break;
651 #if USE(JSVALUE64)
652     case SetTrustedJSConstant:
653         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
654         break;
655     case SetJSConstant:
656         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
657         break;
658     case SetDoubleConstant:
659         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
660         break;
661     case Load32PayloadBoxInt:
662         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
663         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
664         break;
665     case Load32PayloadConvertToInt52:
666         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
667         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
668         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
669         break;
670     case Load32PayloadSignExtend:
671         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
672         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
673         break;
674 #else
675     case SetJSConstantTag:
676         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
677         break;
678     case SetJSConstantPayload:
679         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
680         break;
681     case SetInt32Tag:
682         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
683         break;
684     case SetCellTag:
685         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
686         break;
687     case SetBooleanTag:
688         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
689         break;
690     case SetDoubleConstant:
691         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
692         break;
693 #endif
694     case Load32Tag:
695         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
696         break;
697     case Load32Payload:
698         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
699         break;
700     case LoadPtr:
701         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
702         break;
703 #if USE(JSVALUE64)
704     case Load64:
705         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
706         break;
707     case Load64ShiftInt52Right:
708         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
709         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
710         break;
711     case Load64ShiftInt52Left:
712         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
713         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
714         break;
715 #endif
716     case LoadDouble:
717         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
718         break;
719     default:
720         RELEASE_ASSERT_NOT_REACHED();
721     }
722 }
723     
724 JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, IndexingType shape)
725 {
726     switch (arrayMode.arrayClass()) {
727     case Array::OriginalArray: {
728         CRASH();
729 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
730         JITCompiler::Jump result; // I already know that VC++ takes unkindly to the expression "return Jump()", so I'm doing it this way in anticipation of someone eventually using VC++ to compile the DFG.
731         return result;
732 #endif
733     }
734         
735     case Array::Array:
736         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
737         return m_jit.branch32(
738             MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape));
739         
740     case Array::NonArray:
741     case Array::OriginalNonArray:
742         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
743         return m_jit.branch32(
744             MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
745         
746     case Array::PossiblyArray:
747         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
748         return m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
749     }
750     
751     RELEASE_ASSERT_NOT_REACHED();
752     return JITCompiler::Jump();
753 }
754
755 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
756 {
757     JITCompiler::JumpList result;
758     
759     switch (arrayMode.type()) {
760     case Array::Int32:
761     case Array::Double:
762     case Array::Contiguous:
763     case Array::Undecided:
764         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, arrayMode.shapeMask());
765
766     case Array::ArrayStorage:
767     case Array::SlowPutArrayStorage: {
768         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
769         
770         if (arrayMode.isJSArray()) {
771             if (arrayMode.isSlowPut()) {
772                 result.append(
773                     m_jit.branchTest32(
774                         MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
775                 m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
776                 m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
777                 result.append(
778                     m_jit.branch32(
779                         MacroAssembler::Above, tempGPR,
780                         TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
781                 break;
782             }
783             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
784             result.append(
785                 m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | ArrayStorageShape)));
786             break;
787         }
788         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
789         if (arrayMode.isSlowPut()) {
790             m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
791             result.append(
792                 m_jit.branch32(
793                     MacroAssembler::Above, tempGPR,
794                     TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
795             break;
796         }
797         result.append(
798             m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape)));
799         break;
800     }
801     default:
802         CRASH();
803         break;
804     }
805     
806     return result;
807 }
808
809 void SpeculativeJIT::checkArray(Node* node)
810 {
811     ASSERT(node->arrayMode().isSpecific());
812     ASSERT(!node->arrayMode().doesConversion());
813     
814     SpeculateCellOperand base(this, node->child1());
815     GPRReg baseReg = base.gpr();
816     
817     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
818         noResult(m_currentNode);
819         return;
820     }
821     
822     const ClassInfo* expectedClassInfo = 0;
823     
824     switch (node->arrayMode().type()) {
825     case Array::AnyTypedArray:
826     case Array::String:
827         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
828         break;
829     case Array::Int32:
830     case Array::Double:
831     case Array::Contiguous:
832     case Array::Undecided:
833     case Array::ArrayStorage:
834     case Array::SlowPutArrayStorage: {
835         GPRTemporary temp(this);
836         GPRReg tempGPR = temp.gpr();
837         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
838         speculationCheck(
839             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
840             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
841         
842         noResult(m_currentNode);
843         return;
844     }
845     case Array::DirectArguments:
846         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
847         noResult(m_currentNode);
848         return;
849     case Array::ScopedArguments:
850         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
851         noResult(m_currentNode);
852         return;
853     default:
854         speculateCellTypeWithoutTypeFiltering(
855             node->child1(), baseReg,
856             typeForTypedArrayType(node->arrayMode().typedArrayType()));
857         noResult(m_currentNode);
858         return;
859     }
860     
861     RELEASE_ASSERT(expectedClassInfo);
862     
863     GPRTemporary temp(this);
864     GPRTemporary temp2(this);
865     m_jit.emitLoadStructure(*m_jit.vm(), baseReg, temp.gpr(), temp2.gpr());
866     speculationCheck(
867         BadType, JSValueSource::unboxedCell(baseReg), node,
868         m_jit.branchPtr(
869             MacroAssembler::NotEqual,
870             MacroAssembler::Address(temp.gpr(), Structure::classInfoOffset()),
871             TrustedImmPtr(expectedClassInfo)));
872     
873     noResult(m_currentNode);
874 }
875
876 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
877 {
878     ASSERT(node->arrayMode().doesConversion());
879     
880     GPRTemporary temp(this);
881     GPRTemporary structure;
882     GPRReg tempGPR = temp.gpr();
883     GPRReg structureGPR = InvalidGPRReg;
884     
885     if (node->op() != ArrayifyToStructure) {
886         GPRTemporary realStructure(this);
887         structure.adopt(realStructure);
888         structureGPR = structure.gpr();
889     }
890         
891     // We can skip all that comes next if we already have array storage.
892     MacroAssembler::JumpList slowPath;
893     
894     if (node->op() == ArrayifyToStructure) {
895         slowPath.append(m_jit.branchWeakStructure(
896             JITCompiler::NotEqual,
897             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
898             node->structure()));
899     } else {
900         m_jit.load8(
901             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
902         
903         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
904     }
905     
906     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
907         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
908     
909     noResult(m_currentNode);
910 }
911
912 void SpeculativeJIT::arrayify(Node* node)
913 {
914     ASSERT(node->arrayMode().isSpecific());
915     
916     SpeculateCellOperand base(this, node->child1());
917     
918     if (!node->child2()) {
919         arrayify(node, base.gpr(), InvalidGPRReg);
920         return;
921     }
922     
923     SpeculateInt32Operand property(this, node->child2());
924     
925     arrayify(node, base.gpr(), property.gpr());
926 }
927
928 GPRReg SpeculativeJIT::fillStorage(Edge edge)
929 {
930     VirtualRegister virtualRegister = edge->virtualRegister();
931     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
932     
933     switch (info.registerFormat()) {
934     case DataFormatNone: {
935         if (info.spillFormat() == DataFormatStorage) {
936             GPRReg gpr = allocate();
937             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
938             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
939             info.fillStorage(*m_stream, gpr);
940             return gpr;
941         }
942         
943         // Must be a cell; fill it as a cell and then return the pointer.
944         return fillSpeculateCell(edge);
945     }
946         
947     case DataFormatStorage: {
948         GPRReg gpr = info.gpr();
949         m_gprs.lock(gpr);
950         return gpr;
951     }
952         
953     default:
954         return fillSpeculateCell(edge);
955     }
956 }
957
958 void SpeculativeJIT::useChildren(Node* node)
959 {
960     if (node->flags() & NodeHasVarArgs) {
961         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
962             if (!!m_jit.graph().m_varArgChildren[childIdx])
963                 use(m_jit.graph().m_varArgChildren[childIdx]);
964         }
965     } else {
966         Edge child1 = node->child1();
967         if (!child1) {
968             ASSERT(!node->child2() && !node->child3());
969             return;
970         }
971         use(child1);
972         
973         Edge child2 = node->child2();
974         if (!child2) {
975             ASSERT(!node->child3());
976             return;
977         }
978         use(child2);
979         
980         Edge child3 = node->child3();
981         if (!child3)
982             return;
983         use(child3);
984     }
985 }
986
987 void SpeculativeJIT::compileTryGetById(Node* node)
988 {
989     switch (node->child1().useKind()) {
990     case CellUse: {
991         SpeculateCellOperand base(this, node->child1());
992         JSValueRegsTemporary result(this, Reuse, base);
993
994         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
995         JSValueRegs resultRegs = result.regs();
996
997         base.use();
998
999         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, AccessType::TryGet);
1000
1001         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1002         break;
1003     }
1004
1005     case UntypedUse: {
1006         JSValueOperand base(this, node->child1());
1007         JSValueRegsTemporary result(this, Reuse, base);
1008
1009         JSValueRegs baseRegs = base.jsValueRegs();
1010         JSValueRegs resultRegs = result.regs();
1011
1012         base.use();
1013
1014         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1015
1016         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, AccessType::TryGet);
1017
1018         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1019         break;
1020     }
1021
1022     default:
1023         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1024         break;
1025     } 
1026 }
1027
1028 void SpeculativeJIT::compileIn(Node* node)
1029 {
1030     SpeculateCellOperand base(this, node->child1());
1031     GPRReg baseGPR = base.gpr();
1032     
1033     if (JSString* string = node->child2()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
1034         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1035             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1036             
1037             GPRTemporary result(this);
1038             GPRReg resultGPR = result.gpr();
1039
1040             use(node->child2());
1041             
1042             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1043             MacroAssembler::Label done = m_jit.label();
1044             
1045             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1046             // we can cast it to const AtomicStringImpl* safely.
1047             auto slowPath = slowPathCall(
1048                 jump.m_jump, this, operationInOptimize,
1049                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1050                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1051             
1052             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1053             stubInfo->codeOrigin = node->origin.semantic;
1054             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1055             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1056             stubInfo->patch.thisGPR = static_cast<int8_t>(InvalidGPRReg);
1057 #if USE(JSVALUE32_64)
1058             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1059             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1060             stubInfo->patch.thisTagGPR = static_cast<int8_t>(InvalidGPRReg);
1061 #endif
1062             stubInfo->patch.usedRegisters = usedRegisters();
1063
1064             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1065             addSlowPathGenerator(WTFMove(slowPath));
1066
1067             base.use();
1068
1069             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1070             return;
1071         }
1072     }
1073
1074     JSValueOperand key(this, node->child2());
1075     JSValueRegs regs = key.jsValueRegs();
1076         
1077     GPRFlushedCallResult result(this);
1078     GPRReg resultGPR = result.gpr();
1079         
1080     base.use();
1081     key.use();
1082         
1083     flushRegisters();
1084     callOperation(
1085         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1086         baseGPR, regs);
1087     m_jit.exceptionCheck();
1088     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1089 }
1090
1091 void SpeculativeJIT::compileDeleteById(Node* node)
1092 {
1093     JSValueOperand value(this, node->child1());
1094     GPRFlushedCallResult result(this);
1095
1096     JSValueRegs valueRegs = value.jsValueRegs();
1097     GPRReg resultGPR = result.gpr();
1098
1099     value.use();
1100
1101     flushRegisters();
1102     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1103     m_jit.exceptionCheck();
1104
1105     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1106 }
1107
1108 void SpeculativeJIT::compileDeleteByVal(Node* node)
1109 {
1110     JSValueOperand base(this, node->child1());
1111     JSValueOperand key(this, node->child2());
1112     GPRFlushedCallResult result(this);
1113
1114     JSValueRegs baseRegs = base.jsValueRegs();
1115     JSValueRegs keyRegs = key.jsValueRegs();
1116     GPRReg resultGPR = result.gpr();
1117
1118     base.use();
1119     key.use();
1120
1121     flushRegisters();
1122     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1123     m_jit.exceptionCheck();
1124
1125     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1126 }
1127
1128 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1129 {
1130     unsigned branchIndexInBlock = detectPeepHoleBranch();
1131     if (branchIndexInBlock != UINT_MAX) {
1132         Node* branchNode = m_block->at(branchIndexInBlock);
1133
1134         ASSERT(node->adjustedRefCount() == 1);
1135         
1136         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1137     
1138         m_indexInBlock = branchIndexInBlock;
1139         m_currentNode = branchNode;
1140         
1141         return true;
1142     }
1143     
1144     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1145     
1146     return false;
1147 }
1148
1149 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1150 {
1151     unsigned branchIndexInBlock = detectPeepHoleBranch();
1152     if (branchIndexInBlock != UINT_MAX) {
1153         Node* branchNode = m_block->at(branchIndexInBlock);
1154
1155         ASSERT(node->adjustedRefCount() == 1);
1156         
1157         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1158     
1159         m_indexInBlock = branchIndexInBlock;
1160         m_currentNode = branchNode;
1161         
1162         return true;
1163     }
1164     
1165     nonSpeculativeNonPeepholeStrictEq(node, invert);
1166     
1167     return false;
1168 }
1169
1170 static const char* dataFormatString(DataFormat format)
1171 {
1172     // These values correspond to the DataFormat enum.
1173     const char* strings[] = {
1174         "[  ]",
1175         "[ i]",
1176         "[ d]",
1177         "[ c]",
1178         "Err!",
1179         "Err!",
1180         "Err!",
1181         "Err!",
1182         "[J ]",
1183         "[Ji]",
1184         "[Jd]",
1185         "[Jc]",
1186         "Err!",
1187         "Err!",
1188         "Err!",
1189         "Err!",
1190     };
1191     return strings[format];
1192 }
1193
1194 void SpeculativeJIT::dump(const char* label)
1195 {
1196     if (label)
1197         dataLogF("<%s>\n", label);
1198
1199     dataLogF("  gprs:\n");
1200     m_gprs.dump();
1201     dataLogF("  fprs:\n");
1202     m_fprs.dump();
1203     dataLogF("  VirtualRegisters:\n");
1204     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1205         GenerationInfo& info = m_generationInfo[i];
1206         if (info.alive())
1207             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1208         else
1209             dataLogF("    % 3d:[__][__]", i);
1210         if (info.registerFormat() == DataFormatDouble)
1211             dataLogF(":fpr%d\n", info.fpr());
1212         else if (info.registerFormat() != DataFormatNone
1213 #if USE(JSVALUE32_64)
1214             && !(info.registerFormat() & DataFormatJS)
1215 #endif
1216             ) {
1217             ASSERT(info.gpr() != InvalidGPRReg);
1218             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1219         } else
1220             dataLogF("\n");
1221     }
1222     if (label)
1223         dataLogF("</%s>\n", label);
1224 }
1225
1226 GPRTemporary::GPRTemporary()
1227     : m_jit(0)
1228     , m_gpr(InvalidGPRReg)
1229 {
1230 }
1231
1232 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1233     : m_jit(jit)
1234     , m_gpr(InvalidGPRReg)
1235 {
1236     m_gpr = m_jit->allocate();
1237 }
1238
1239 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1240     : m_jit(jit)
1241     , m_gpr(InvalidGPRReg)
1242 {
1243     m_gpr = m_jit->allocate(specific);
1244 }
1245
1246 #if USE(JSVALUE32_64)
1247 GPRTemporary::GPRTemporary(
1248     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1249     : m_jit(jit)
1250     , m_gpr(InvalidGPRReg)
1251 {
1252     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1253         m_gpr = m_jit->reuse(op1.gpr(which));
1254     else
1255         m_gpr = m_jit->allocate();
1256 }
1257 #endif // USE(JSVALUE32_64)
1258
1259 JSValueRegsTemporary::JSValueRegsTemporary() { }
1260
1261 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1262 #if USE(JSVALUE64)
1263     : m_gpr(jit)
1264 #else
1265     : m_payloadGPR(jit)
1266     , m_tagGPR(jit)
1267 #endif
1268 {
1269 }
1270
1271 #if USE(JSVALUE64)
1272 template<typename T>
1273 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1274     : m_gpr(jit, Reuse, operand)
1275 {
1276 }
1277 #else
1278 template<typename T>
1279 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1280 {
1281     if (resultWord == PayloadWord) {
1282         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1283         m_tagGPR = GPRTemporary(jit);
1284     } else {
1285         m_payloadGPR = GPRTemporary(jit);
1286         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1287     }
1288 }
1289 #endif
1290
1291 #if USE(JSVALUE64)
1292 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1293 {
1294     m_gpr = GPRTemporary(jit, Reuse, operand);
1295 }
1296 #else
1297 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1298 {
1299     if (jit->canReuse(operand.node())) {
1300         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1301         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1302     } else {
1303         m_payloadGPR = GPRTemporary(jit);
1304         m_tagGPR = GPRTemporary(jit);
1305     }
1306 }
1307 #endif
1308
1309 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1310
1311 JSValueRegs JSValueRegsTemporary::regs()
1312 {
1313 #if USE(JSVALUE64)
1314     return JSValueRegs(m_gpr.gpr());
1315 #else
1316     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1317 #endif
1318 }
1319
1320 void GPRTemporary::adopt(GPRTemporary& other)
1321 {
1322     ASSERT(!m_jit);
1323     ASSERT(m_gpr == InvalidGPRReg);
1324     ASSERT(other.m_jit);
1325     ASSERT(other.m_gpr != InvalidGPRReg);
1326     m_jit = other.m_jit;
1327     m_gpr = other.m_gpr;
1328     other.m_jit = 0;
1329     other.m_gpr = InvalidGPRReg;
1330 }
1331
1332 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1333 {
1334     ASSERT(other.m_jit);
1335     ASSERT(other.m_fpr != InvalidFPRReg);
1336     m_jit = other.m_jit;
1337     m_fpr = other.m_fpr;
1338
1339     other.m_jit = nullptr;
1340 }
1341
1342 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1343     : m_jit(jit)
1344     , m_fpr(InvalidFPRReg)
1345 {
1346     m_fpr = m_jit->fprAllocate();
1347 }
1348
1349 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1350     : m_jit(jit)
1351     , m_fpr(InvalidFPRReg)
1352 {
1353     if (m_jit->canReuse(op1.node()))
1354         m_fpr = m_jit->reuse(op1.fpr());
1355     else
1356         m_fpr = m_jit->fprAllocate();
1357 }
1358
1359 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1360     : m_jit(jit)
1361     , m_fpr(InvalidFPRReg)
1362 {
1363     if (m_jit->canReuse(op1.node()))
1364         m_fpr = m_jit->reuse(op1.fpr());
1365     else if (m_jit->canReuse(op2.node()))
1366         m_fpr = m_jit->reuse(op2.fpr());
1367     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1368         m_fpr = m_jit->reuse(op1.fpr());
1369     else
1370         m_fpr = m_jit->fprAllocate();
1371 }
1372
1373 #if USE(JSVALUE32_64)
1374 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1375     : m_jit(jit)
1376     , m_fpr(InvalidFPRReg)
1377 {
1378     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1379         m_fpr = m_jit->reuse(op1.fpr());
1380     else
1381         m_fpr = m_jit->fprAllocate();
1382 }
1383 #endif
1384
1385 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1386 {
1387     BasicBlock* taken = branchNode->branchData()->taken.block;
1388     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1389
1390     if (taken == nextBlock()) {
1391         condition = MacroAssembler::invert(condition);
1392         std::swap(taken, notTaken);
1393     }
1394
1395     SpeculateDoubleOperand op1(this, node->child1());
1396     SpeculateDoubleOperand op2(this, node->child2());
1397     
1398     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1399     jump(notTaken);
1400 }
1401
1402 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1403 {
1404     BasicBlock* taken = branchNode->branchData()->taken.block;
1405     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1406
1407     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1408     
1409     if (taken == nextBlock()) {
1410         condition = MacroAssembler::NotEqual;
1411         BasicBlock* tmp = taken;
1412         taken = notTaken;
1413         notTaken = tmp;
1414     }
1415
1416     SpeculateCellOperand op1(this, node->child1());
1417     SpeculateCellOperand op2(this, node->child2());
1418     
1419     GPRReg op1GPR = op1.gpr();
1420     GPRReg op2GPR = op2.gpr();
1421     
1422     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1423         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1424             speculationCheck(
1425                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1426         }
1427         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1428             speculationCheck(
1429                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1430         }
1431     } else {
1432         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1433             speculationCheck(
1434                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1435                 m_jit.branchIfNotObject(op1GPR));
1436         }
1437         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1438             m_jit.branchTest8(
1439                 MacroAssembler::NonZero, 
1440                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1441                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1442
1443         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1444             speculationCheck(
1445                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1446                 m_jit.branchIfNotObject(op2GPR));
1447         }
1448         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1449             m_jit.branchTest8(
1450                 MacroAssembler::NonZero, 
1451                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1452                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1453     }
1454
1455     branchPtr(condition, op1GPR, op2GPR, taken);
1456     jump(notTaken);
1457 }
1458
1459 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1460 {
1461     BasicBlock* taken = branchNode->branchData()->taken.block;
1462     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1463
1464     // The branch instruction will branch to the taken block.
1465     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1466     if (taken == nextBlock()) {
1467         condition = JITCompiler::invert(condition);
1468         BasicBlock* tmp = taken;
1469         taken = notTaken;
1470         notTaken = tmp;
1471     }
1472
1473     if (node->child1()->isInt32Constant()) {
1474         int32_t imm = node->child1()->asInt32();
1475         SpeculateBooleanOperand op2(this, node->child2());
1476         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1477     } else if (node->child2()->isInt32Constant()) {
1478         SpeculateBooleanOperand op1(this, node->child1());
1479         int32_t imm = node->child2()->asInt32();
1480         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1481     } else {
1482         SpeculateBooleanOperand op1(this, node->child1());
1483         SpeculateBooleanOperand op2(this, node->child2());
1484         branch32(condition, op1.gpr(), op2.gpr(), taken);
1485     }
1486
1487     jump(notTaken);
1488 }
1489
1490 void SpeculativeJIT::compileToLowerCase(Node* node)
1491 {
1492     ASSERT(node->op() == ToLowerCase);
1493     SpeculateCellOperand string(this, node->child1());
1494     GPRTemporary temp(this);
1495     GPRTemporary index(this);
1496     GPRTemporary charReg(this);
1497     GPRTemporary length(this);
1498
1499     GPRReg stringGPR = string.gpr();
1500     GPRReg tempGPR = temp.gpr();
1501     GPRReg indexGPR = index.gpr();
1502     GPRReg charGPR = charReg.gpr();
1503     GPRReg lengthGPR = length.gpr();
1504
1505     speculateString(node->child1(), stringGPR);
1506
1507     CCallHelpers::JumpList slowPath;
1508
1509     m_jit.move(TrustedImmPtr(0), indexGPR);
1510
1511     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1512     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1513
1514     slowPath.append(m_jit.branchTest32(
1515         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1516         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1517     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1518     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1519
1520     auto loopStart = m_jit.label();
1521     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1522     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1523     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1524     m_jit.sub32(TrustedImm32('A'), charGPR);
1525     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1526
1527     m_jit.add32(TrustedImm32(1), indexGPR);
1528     m_jit.jump().linkTo(loopStart, &m_jit);
1529     
1530     slowPath.link(&m_jit);
1531     silentSpillAllRegisters(lengthGPR);
1532     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1533     silentFillAllRegisters();
1534     m_jit.exceptionCheck();
1535     auto done = m_jit.jump();
1536
1537     loopDone.link(&m_jit);
1538     m_jit.move(stringGPR, lengthGPR);
1539
1540     done.link(&m_jit);
1541     cellResult(lengthGPR, node);
1542 }
1543
1544 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1545 {
1546     BasicBlock* taken = branchNode->branchData()->taken.block;
1547     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1548
1549     // The branch instruction will branch to the taken block.
1550     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1551     if (taken == nextBlock()) {
1552         condition = JITCompiler::invert(condition);
1553         BasicBlock* tmp = taken;
1554         taken = notTaken;
1555         notTaken = tmp;
1556     }
1557
1558     if (node->child1()->isInt32Constant()) {
1559         int32_t imm = node->child1()->asInt32();
1560         SpeculateInt32Operand op2(this, node->child2());
1561         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1562     } else if (node->child2()->isInt32Constant()) {
1563         SpeculateInt32Operand op1(this, node->child1());
1564         int32_t imm = node->child2()->asInt32();
1565         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1566     } else {
1567         SpeculateInt32Operand op1(this, node->child1());
1568         SpeculateInt32Operand op2(this, node->child2());
1569         branch32(condition, op1.gpr(), op2.gpr(), taken);
1570     }
1571
1572     jump(notTaken);
1573 }
1574
1575 // Returns true if the compare is fused with a subsequent branch.
1576 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1577 {
1578     // Fused compare & branch.
1579     unsigned branchIndexInBlock = detectPeepHoleBranch();
1580     if (branchIndexInBlock != UINT_MAX) {
1581         Node* branchNode = m_block->at(branchIndexInBlock);
1582
1583         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1584         // so can be no intervening nodes to also reference the compare. 
1585         ASSERT(node->adjustedRefCount() == 1);
1586
1587         if (node->isBinaryUseKind(Int32Use))
1588             compilePeepHoleInt32Branch(node, branchNode, condition);
1589 #if USE(JSVALUE64)
1590         else if (node->isBinaryUseKind(Int52RepUse))
1591             compilePeepHoleInt52Branch(node, branchNode, condition);
1592 #endif // USE(JSVALUE64)
1593         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1594             // Use non-peephole comparison, for now.
1595             return false;
1596         } else if (node->isBinaryUseKind(DoubleRepUse))
1597             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1598         else if (node->op() == CompareEq) {
1599             if (node->isBinaryUseKind(BooleanUse))
1600                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1601             else if (node->isBinaryUseKind(SymbolUse))
1602                 compilePeepHoleSymbolEquality(node, branchNode);
1603             else if (node->isBinaryUseKind(ObjectUse))
1604                 compilePeepHoleObjectEquality(node, branchNode);
1605             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1606                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1607             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1608                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1609             else if (!needsTypeCheck(node->child1(), SpecOther))
1610                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1611             else if (!needsTypeCheck(node->child2(), SpecOther))
1612                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1613             else {
1614                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1615                 return true;
1616             }
1617         } else {
1618             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1619             return true;
1620         }
1621
1622         use(node->child1());
1623         use(node->child2());
1624         m_indexInBlock = branchIndexInBlock;
1625         m_currentNode = branchNode;
1626         return true;
1627     }
1628     return false;
1629 }
1630
1631 void SpeculativeJIT::noticeOSRBirth(Node* node)
1632 {
1633     if (!node->hasVirtualRegister())
1634         return;
1635     
1636     VirtualRegister virtualRegister = node->virtualRegister();
1637     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1638     
1639     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1640 }
1641
1642 void SpeculativeJIT::compileMovHint(Node* node)
1643 {
1644     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1645     
1646     Node* child = node->child1().node();
1647     noticeOSRBirth(child);
1648     
1649     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1650 }
1651
1652 void SpeculativeJIT::bail(AbortReason reason)
1653 {
1654     if (verboseCompilationEnabled())
1655         dataLog("Bailing compilation.\n");
1656     m_compileOkay = true;
1657     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1658     clearGenerationInfo();
1659 }
1660
1661 void SpeculativeJIT::compileCurrentBlock()
1662 {
1663     ASSERT(m_compileOkay);
1664     
1665     if (!m_block)
1666         return;
1667     
1668     ASSERT(m_block->isReachable);
1669     
1670     m_jit.blockHeads()[m_block->index] = m_jit.label();
1671
1672     if (!m_block->intersectionOfCFAHasVisited) {
1673         // Don't generate code for basic blocks that are unreachable according to CFA.
1674         // But to be sure that nobody has generated a jump to this block, drop in a
1675         // breakpoint here.
1676         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1677         return;
1678     }
1679
1680     m_stream->appendAndLog(VariableEvent::reset());
1681     
1682     m_jit.jitAssertHasValidCallFrame();
1683     m_jit.jitAssertTagsInPlace();
1684     m_jit.jitAssertArgumentCountSane();
1685
1686     m_state.reset();
1687     m_state.beginBasicBlock(m_block);
1688     
1689     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1690         int operand = m_block->variablesAtHead.operandForIndex(i);
1691         Node* node = m_block->variablesAtHead[i];
1692         if (!node)
1693             continue; // No need to record dead SetLocal's.
1694         
1695         VariableAccessData* variable = node->variableAccessData();
1696         DataFormat format;
1697         if (!node->refCount())
1698             continue; // No need to record dead SetLocal's.
1699         format = dataFormatFor(variable->flushFormat());
1700         m_stream->appendAndLog(
1701             VariableEvent::setLocal(
1702                 VirtualRegister(operand),
1703                 variable->machineLocal(),
1704                 format));
1705     }
1706
1707     m_origin = NodeOrigin();
1708     
1709     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1710         m_currentNode = m_block->at(m_indexInBlock);
1711         
1712         // We may have hit a contradiction that the CFA was aware of but that the JIT
1713         // didn't cause directly.
1714         if (!m_state.isValid()) {
1715             bail(DFGBailedAtTopOfBlock);
1716             return;
1717         }
1718
1719         m_interpreter.startExecuting();
1720         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1721         m_jit.setForNode(m_currentNode);
1722         m_origin = m_currentNode->origin;
1723         if (validationEnabled())
1724             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1725         m_lastGeneratedNode = m_currentNode->op();
1726         
1727         ASSERT(m_currentNode->shouldGenerate());
1728         
1729         if (verboseCompilationEnabled()) {
1730             dataLogF(
1731                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1732                 (int)m_currentNode->index(),
1733                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1734             dataLog("\n");
1735         }
1736
1737         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1738             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1739
1740         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1741
1742         compile(m_currentNode);
1743         
1744         if (belongsInMinifiedGraph(m_currentNode->op()))
1745             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1746         
1747 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1748         m_jit.clearRegisterAllocationOffsets();
1749 #endif
1750         
1751         if (!m_compileOkay) {
1752             bail(DFGBailedAtEndOfNode);
1753             return;
1754         }
1755         
1756         // Make sure that the abstract state is rematerialized for the next node.
1757         m_interpreter.executeEffects(m_indexInBlock);
1758     }
1759     
1760     // Perform the most basic verification that children have been used correctly.
1761     if (!ASSERT_DISABLED) {
1762         for (auto& info : m_generationInfo)
1763             RELEASE_ASSERT(!info.alive());
1764     }
1765 }
1766
1767 // If we are making type predictions about our arguments then
1768 // we need to check that they are correct on function entry.
1769 void SpeculativeJIT::checkArgumentTypes()
1770 {
1771     ASSERT(!m_currentNode);
1772     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1773
1774     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1775         Node* node = m_jit.graph().m_arguments[i];
1776         if (!node) {
1777             // The argument is dead. We don't do any checks for such arguments.
1778             continue;
1779         }
1780         
1781         ASSERT(node->op() == SetArgument);
1782         ASSERT(node->shouldGenerate());
1783
1784         VariableAccessData* variableAccessData = node->variableAccessData();
1785         FlushFormat format = variableAccessData->flushFormat();
1786         
1787         if (format == FlushedJSValue)
1788             continue;
1789         
1790         VirtualRegister virtualRegister = variableAccessData->local();
1791
1792         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1793         
1794 #if USE(JSVALUE64)
1795         switch (format) {
1796         case FlushedInt32: {
1797             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1798             break;
1799         }
1800         case FlushedBoolean: {
1801             GPRTemporary temp(this);
1802             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1803             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1804             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1805             break;
1806         }
1807         case FlushedCell: {
1808             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1809             break;
1810         }
1811         default:
1812             RELEASE_ASSERT_NOT_REACHED();
1813             break;
1814         }
1815 #else
1816         switch (format) {
1817         case FlushedInt32: {
1818             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1819             break;
1820         }
1821         case FlushedBoolean: {
1822             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1823             break;
1824         }
1825         case FlushedCell: {
1826             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1827             break;
1828         }
1829         default:
1830             RELEASE_ASSERT_NOT_REACHED();
1831             break;
1832         }
1833 #endif
1834     }
1835
1836     m_origin = NodeOrigin();
1837 }
1838
1839 bool SpeculativeJIT::compile()
1840 {
1841     checkArgumentTypes();
1842     
1843     ASSERT(!m_currentNode);
1844     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1845         m_jit.setForBlockIndex(blockIndex);
1846         m_block = m_jit.graph().block(blockIndex);
1847         compileCurrentBlock();
1848     }
1849     linkBranches();
1850     return true;
1851 }
1852
1853 void SpeculativeJIT::createOSREntries()
1854 {
1855     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1856         BasicBlock* block = m_jit.graph().block(blockIndex);
1857         if (!block)
1858             continue;
1859         if (!block->isOSRTarget)
1860             continue;
1861         
1862         // Currently we don't have OSR entry trampolines. We could add them
1863         // here if need be.
1864         m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1865     }
1866 }
1867
1868 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1869 {
1870     unsigned osrEntryIndex = 0;
1871     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1872         BasicBlock* block = m_jit.graph().block(blockIndex);
1873         if (!block)
1874             continue;
1875         if (!block->isOSRTarget)
1876             continue;
1877         m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
1878     }
1879     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
1880     
1881     if (verboseCompilationEnabled()) {
1882         DumpContext dumpContext;
1883         dataLog("OSR Entries:\n");
1884         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
1885             dataLog("    ", inContext(entryData, &dumpContext), "\n");
1886         if (!dumpContext.isEmpty())
1887             dumpContext.dump(WTF::dataFile());
1888     }
1889 }
1890     
1891 void SpeculativeJIT::compileCheckTraps(Node*)
1892 {
1893     ASSERT(Options::usePollingTraps());
1894     GPRTemporary unused(this);
1895     GPRReg unusedGPR = unused.gpr();
1896
1897     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
1898         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
1899
1900     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
1901 }
1902
1903 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
1904 {
1905     Edge child3 = m_jit.graph().varArgChild(node, 2);
1906     Edge child4 = m_jit.graph().varArgChild(node, 3);
1907
1908     ArrayMode arrayMode = node->arrayMode();
1909     
1910     GPRReg baseReg = base.gpr();
1911     GPRReg propertyReg = property.gpr();
1912     
1913     SpeculateDoubleOperand value(this, child3);
1914
1915     FPRReg valueReg = value.fpr();
1916     
1917     DFG_TYPE_CHECK(
1918         JSValueRegs(), child3, SpecFullRealNumber,
1919         m_jit.branchDouble(
1920             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
1921     
1922     if (!m_compileOkay)
1923         return;
1924     
1925     StorageOperand storage(this, child4);
1926     GPRReg storageReg = storage.gpr();
1927
1928     if (node->op() == PutByValAlias) {
1929         // Store the value to the array.
1930         GPRReg propertyReg = property.gpr();
1931         FPRReg valueReg = value.fpr();
1932         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
1933         
1934         noResult(m_currentNode);
1935         return;
1936     }
1937     
1938     GPRTemporary temporary;
1939     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
1940
1941     MacroAssembler::Jump slowCase;
1942     
1943     if (arrayMode.isInBounds()) {
1944         speculationCheck(
1945             OutOfBounds, JSValueRegs(), 0,
1946             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
1947     } else {
1948         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1949         
1950         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
1951         
1952         if (!arrayMode.isOutOfBounds())
1953             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
1954         
1955         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
1956         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1957         
1958         inBounds.link(&m_jit);
1959     }
1960     
1961     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
1962
1963     base.use();
1964     property.use();
1965     value.use();
1966     storage.use();
1967     
1968     if (arrayMode.isOutOfBounds()) {
1969         addSlowPathGenerator(
1970             slowPathCall(
1971                 slowCase, this,
1972                 m_jit.codeBlock()->isStrictMode() ? operationPutDoubleByValBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1973                 NoResult, baseReg, propertyReg, valueReg));
1974     }
1975
1976     noResult(m_currentNode, UseChildrenCalledExplicitly);
1977 }
1978
1979 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
1980 {
1981     SpeculateCellOperand string(this, node->child1());
1982     SpeculateStrictInt32Operand index(this, node->child2());
1983     StorageOperand storage(this, node->child3());
1984
1985     GPRReg stringReg = string.gpr();
1986     GPRReg indexReg = index.gpr();
1987     GPRReg storageReg = storage.gpr();
1988     
1989     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
1990
1991     // unsigned comparison so we can filter out negative indices and indices that are too large
1992     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
1993
1994     GPRTemporary scratch(this);
1995     GPRReg scratchReg = scratch.gpr();
1996
1997     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
1998
1999     // Load the character into scratchReg
2000     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2001
2002     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2003     JITCompiler::Jump cont8Bit = m_jit.jump();
2004
2005     is16Bit.link(&m_jit);
2006
2007     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2008
2009     cont8Bit.link(&m_jit);
2010
2011     int32Result(scratchReg, m_currentNode);
2012 }
2013
2014 void SpeculativeJIT::compileGetByValOnString(Node* node)
2015 {
2016     SpeculateCellOperand base(this, node->child1());
2017     SpeculateStrictInt32Operand property(this, node->child2());
2018     StorageOperand storage(this, node->child3());
2019     GPRReg baseReg = base.gpr();
2020     GPRReg propertyReg = property.gpr();
2021     GPRReg storageReg = storage.gpr();
2022
2023     GPRTemporary scratch(this);
2024     GPRReg scratchReg = scratch.gpr();
2025 #if USE(JSVALUE32_64)
2026     GPRTemporary resultTag;
2027     GPRReg resultTagReg = InvalidGPRReg;
2028     if (node->arrayMode().isOutOfBounds()) {
2029         GPRTemporary realResultTag(this);
2030         resultTag.adopt(realResultTag);
2031         resultTagReg = resultTag.gpr();
2032     }
2033 #endif
2034
2035     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2036
2037     // unsigned comparison so we can filter out negative indices and indices that are too large
2038     JITCompiler::Jump outOfBounds = m_jit.branch32(
2039         MacroAssembler::AboveOrEqual, propertyReg,
2040         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2041     if (node->arrayMode().isInBounds())
2042         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2043
2044     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2045
2046     // Load the character into scratchReg
2047     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2048
2049     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2050     JITCompiler::Jump cont8Bit = m_jit.jump();
2051
2052     is16Bit.link(&m_jit);
2053
2054     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2055
2056     JITCompiler::Jump bigCharacter =
2057         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2058
2059     // 8 bit string values don't need the isASCII check.
2060     cont8Bit.link(&m_jit);
2061
2062     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2063     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2064     m_jit.loadPtr(scratchReg, scratchReg);
2065
2066     addSlowPathGenerator(
2067         slowPathCall(
2068             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2069
2070     if (node->arrayMode().isOutOfBounds()) {
2071 #if USE(JSVALUE32_64)
2072         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2073 #endif
2074
2075         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2076         bool prototypeChainIsSane = false;
2077         if (globalObject->stringPrototypeChainIsSane()) {
2078             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2079             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2080             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2081             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2082             // indexed properties either.
2083             // https://bugs.webkit.org/show_bug.cgi?id=144668
2084             m_jit.graph().watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
2085             m_jit.graph().watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
2086             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2087         }
2088         if (prototypeChainIsSane) {
2089             m_jit.graph().watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
2090             m_jit.graph().watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
2091             
2092 #if USE(JSVALUE64)
2093             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2094                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2095 #else
2096             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2097                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2098                 baseReg, propertyReg));
2099 #endif
2100         } else {
2101 #if USE(JSVALUE64)
2102             addSlowPathGenerator(
2103                 slowPathCall(
2104                     outOfBounds, this, operationGetByValStringInt,
2105                     scratchReg, baseReg, propertyReg));
2106 #else
2107             addSlowPathGenerator(
2108                 slowPathCall(
2109                     outOfBounds, this, operationGetByValStringInt,
2110                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2111 #endif
2112         }
2113         
2114 #if USE(JSVALUE64)
2115         jsValueResult(scratchReg, m_currentNode);
2116 #else
2117         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2118 #endif
2119     } else
2120         cellResult(scratchReg, m_currentNode);
2121 }
2122
2123 void SpeculativeJIT::compileFromCharCode(Node* node)
2124 {
2125     Edge& child = node->child1();
2126     if (child.useKind() == UntypedUse) {
2127         JSValueOperand opr(this, child);
2128         JSValueRegs oprRegs = opr.jsValueRegs();
2129 #if USE(JSVALUE64)
2130         GPRTemporary result(this);
2131         JSValueRegs resultRegs = JSValueRegs(result.gpr());
2132 #else
2133         GPRTemporary resultTag(this);
2134         GPRTemporary resultPayload(this);
2135         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
2136 #endif
2137         flushRegisters();
2138         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2139         m_jit.exceptionCheck();
2140         
2141         jsValueResult(resultRegs, node);
2142         return;
2143     }
2144
2145     SpeculateStrictInt32Operand property(this, child);
2146     GPRReg propertyReg = property.gpr();
2147     GPRTemporary smallStrings(this);
2148     GPRTemporary scratch(this);
2149     GPRReg scratchReg = scratch.gpr();
2150     GPRReg smallStringsReg = smallStrings.gpr();
2151
2152     JITCompiler::JumpList slowCases;
2153     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2154     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2155     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2156
2157     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2158     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2159     cellResult(scratchReg, m_currentNode);
2160 }
2161
2162 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2163 {
2164     VirtualRegister virtualRegister = node->virtualRegister();
2165     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2166
2167     switch (info.registerFormat()) {
2168     case DataFormatStorage:
2169         RELEASE_ASSERT_NOT_REACHED();
2170
2171     case DataFormatBoolean:
2172     case DataFormatCell:
2173         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2174         return GeneratedOperandTypeUnknown;
2175
2176     case DataFormatNone:
2177     case DataFormatJSCell:
2178     case DataFormatJS:
2179     case DataFormatJSBoolean:
2180     case DataFormatJSDouble:
2181         return GeneratedOperandJSValue;
2182
2183     case DataFormatJSInt32:
2184     case DataFormatInt32:
2185         return GeneratedOperandInteger;
2186
2187     default:
2188         RELEASE_ASSERT_NOT_REACHED();
2189         return GeneratedOperandTypeUnknown;
2190     }
2191 }
2192
2193 void SpeculativeJIT::compileValueToInt32(Node* node)
2194 {
2195     switch (node->child1().useKind()) {
2196 #if USE(JSVALUE64)
2197     case Int52RepUse: {
2198         SpeculateStrictInt52Operand op1(this, node->child1());
2199         GPRTemporary result(this, Reuse, op1);
2200         GPRReg op1GPR = op1.gpr();
2201         GPRReg resultGPR = result.gpr();
2202         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2203         int32Result(resultGPR, node, DataFormatInt32);
2204         return;
2205     }
2206 #endif // USE(JSVALUE64)
2207         
2208     case DoubleRepUse: {
2209         GPRTemporary result(this);
2210         SpeculateDoubleOperand op1(this, node->child1());
2211         FPRReg fpr = op1.fpr();
2212         GPRReg gpr = result.gpr();
2213         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2214         
2215         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2216             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2217         
2218         int32Result(gpr, node);
2219         return;
2220     }
2221     
2222     case NumberUse:
2223     case NotCellUse: {
2224         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2225         case GeneratedOperandInteger: {
2226             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2227             GPRTemporary result(this, Reuse, op1);
2228             m_jit.move(op1.gpr(), result.gpr());
2229             int32Result(result.gpr(), node, op1.format());
2230             return;
2231         }
2232         case GeneratedOperandJSValue: {
2233             GPRTemporary result(this);
2234 #if USE(JSVALUE64)
2235             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2236
2237             GPRReg gpr = op1.gpr();
2238             GPRReg resultGpr = result.gpr();
2239             FPRTemporary tempFpr(this);
2240             FPRReg fpr = tempFpr.fpr();
2241
2242             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2243             JITCompiler::JumpList converted;
2244
2245             if (node->child1().useKind() == NumberUse) {
2246                 DFG_TYPE_CHECK(
2247                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2248                     m_jit.branchTest64(
2249                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2250             } else {
2251                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2252                 
2253                 DFG_TYPE_CHECK(
2254                     JSValueRegs(gpr), node->child1(), ~SpecCell, m_jit.branchIfCell(JSValueRegs(gpr)));
2255                 
2256                 // It's not a cell: so true turns into 1 and all else turns into 0.
2257                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2258                 converted.append(m_jit.jump());
2259                 
2260                 isNumber.link(&m_jit);
2261             }
2262
2263             // First, if we get here we have a double encoded as a JSValue
2264             unboxDouble(gpr, resultGpr, fpr);
2265
2266             silentSpillAllRegisters(resultGpr);
2267             callOperation(operationToInt32, resultGpr, fpr);
2268             silentFillAllRegisters();
2269
2270             converted.append(m_jit.jump());
2271
2272             isInteger.link(&m_jit);
2273             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2274
2275             converted.link(&m_jit);
2276 #else
2277             Node* childNode = node->child1().node();
2278             VirtualRegister virtualRegister = childNode->virtualRegister();
2279             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2280
2281             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2282
2283             GPRReg payloadGPR = op1.payloadGPR();
2284             GPRReg resultGpr = result.gpr();
2285         
2286             JITCompiler::JumpList converted;
2287
2288             if (info.registerFormat() == DataFormatJSInt32)
2289                 m_jit.move(payloadGPR, resultGpr);
2290             else {
2291                 GPRReg tagGPR = op1.tagGPR();
2292                 FPRTemporary tempFpr(this);
2293                 FPRReg fpr = tempFpr.fpr();
2294                 FPRTemporary scratch(this);
2295
2296                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2297
2298                 if (node->child1().useKind() == NumberUse) {
2299                     DFG_TYPE_CHECK(
2300                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2301                         m_jit.branch32(
2302                             MacroAssembler::AboveOrEqual, tagGPR,
2303                             TrustedImm32(JSValue::LowestTag)));
2304                 } else {
2305                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2306                     
2307                     DFG_TYPE_CHECK(
2308                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2309                         m_jit.branchIfCell(op1.jsValueRegs()));
2310                     
2311                     // It's not a cell: so true turns into 1 and all else turns into 0.
2312                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2313                     m_jit.move(TrustedImm32(0), resultGpr);
2314                     converted.append(m_jit.jump());
2315                     
2316                     isBoolean.link(&m_jit);
2317                     m_jit.move(payloadGPR, resultGpr);
2318                     converted.append(m_jit.jump());
2319                     
2320                     isNumber.link(&m_jit);
2321                 }
2322
2323                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2324
2325                 silentSpillAllRegisters(resultGpr);
2326                 callOperation(operationToInt32, resultGpr, fpr);
2327                 silentFillAllRegisters();
2328
2329                 converted.append(m_jit.jump());
2330
2331                 isInteger.link(&m_jit);
2332                 m_jit.move(payloadGPR, resultGpr);
2333
2334                 converted.link(&m_jit);
2335             }
2336 #endif
2337             int32Result(resultGpr, node);
2338             return;
2339         }
2340         case GeneratedOperandTypeUnknown:
2341             RELEASE_ASSERT(!m_compileOkay);
2342             return;
2343         }
2344         RELEASE_ASSERT_NOT_REACHED();
2345         return;
2346     }
2347     
2348     default:
2349         ASSERT(!m_compileOkay);
2350         return;
2351     }
2352 }
2353
2354 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2355 {
2356     if (doesOverflow(node->arithMode())) {
2357         if (enableInt52()) {
2358             SpeculateInt32Operand op1(this, node->child1());
2359             GPRTemporary result(this, Reuse, op1);
2360             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2361             strictInt52Result(result.gpr(), node);
2362             return;
2363         }
2364         SpeculateInt32Operand op1(this, node->child1());
2365         FPRTemporary result(this);
2366             
2367         GPRReg inputGPR = op1.gpr();
2368         FPRReg outputFPR = result.fpr();
2369             
2370         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2371             
2372         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2373         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2374         positive.link(&m_jit);
2375             
2376         doubleResult(outputFPR, node);
2377         return;
2378     }
2379     
2380     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2381
2382     SpeculateInt32Operand op1(this, node->child1());
2383     GPRTemporary result(this);
2384
2385     m_jit.move(op1.gpr(), result.gpr());
2386
2387     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2388
2389     int32Result(result.gpr(), node, op1.format());
2390 }
2391
2392 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2393 {
2394     SpeculateDoubleOperand op1(this, node->child1());
2395     FPRTemporary scratch(this);
2396     GPRTemporary result(this);
2397     
2398     FPRReg valueFPR = op1.fpr();
2399     FPRReg scratchFPR = scratch.fpr();
2400     GPRReg resultGPR = result.gpr();
2401
2402     JITCompiler::JumpList failureCases;
2403     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2404     m_jit.branchConvertDoubleToInt32(
2405         valueFPR, resultGPR, failureCases, scratchFPR,
2406         shouldCheckNegativeZero(node->arithMode()));
2407     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2408
2409     int32Result(resultGPR, node);
2410 }
2411
2412 void SpeculativeJIT::compileDoubleRep(Node* node)
2413 {
2414     switch (node->child1().useKind()) {
2415     case RealNumberUse: {
2416         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2417         FPRTemporary result(this);
2418         
2419         JSValueRegs op1Regs = op1.jsValueRegs();
2420         FPRReg resultFPR = result.fpr();
2421         
2422 #if USE(JSVALUE64)
2423         GPRTemporary temp(this);
2424         GPRReg tempGPR = temp.gpr();
2425         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2426 #else
2427         FPRTemporary temp(this);
2428         FPRReg tempFPR = temp.fpr();
2429         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2430 #endif
2431         
2432         JITCompiler::Jump done = m_jit.branchDouble(
2433             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2434         
2435         DFG_TYPE_CHECK(
2436             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2437         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2438         
2439         done.link(&m_jit);
2440         
2441         doubleResult(resultFPR, node);
2442         return;
2443     }
2444     
2445     case NotCellUse:
2446     case NumberUse: {
2447         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2448
2449         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2450         if (isInt32Speculation(possibleTypes)) {
2451             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2452             FPRTemporary result(this);
2453             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2454             doubleResult(result.fpr(), node);
2455             return;
2456         }
2457
2458         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2459         FPRTemporary result(this);
2460
2461 #if USE(JSVALUE64)
2462         GPRTemporary temp(this);
2463
2464         GPRReg op1GPR = op1.gpr();
2465         GPRReg tempGPR = temp.gpr();
2466         FPRReg resultFPR = result.fpr();
2467         JITCompiler::JumpList done;
2468
2469         JITCompiler::Jump isInteger = m_jit.branch64(
2470             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2471
2472         if (node->child1().useKind() == NotCellUse) {
2473             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2474             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2475
2476             static const double zero = 0;
2477             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2478
2479             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2480             done.append(isNull);
2481
2482             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCell,
2483                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2484
2485             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2486             static const double one = 1;
2487             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2488             done.append(m_jit.jump());
2489             done.append(isFalse);
2490
2491             isUndefined.link(&m_jit);
2492             static const double NaN = PNaN;
2493             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2494             done.append(m_jit.jump());
2495
2496             isNumber.link(&m_jit);
2497         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2498             typeCheck(
2499                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2500                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2501         }
2502
2503         unboxDouble(op1GPR, tempGPR, resultFPR);
2504         done.append(m_jit.jump());
2505     
2506         isInteger.link(&m_jit);
2507         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2508         done.link(&m_jit);
2509 #else // USE(JSVALUE64) -> this is the 32_64 case
2510         FPRTemporary temp(this);
2511     
2512         GPRReg op1TagGPR = op1.tagGPR();
2513         GPRReg op1PayloadGPR = op1.payloadGPR();
2514         FPRReg tempFPR = temp.fpr();
2515         FPRReg resultFPR = result.fpr();
2516         JITCompiler::JumpList done;
2517     
2518         JITCompiler::Jump isInteger = m_jit.branch32(
2519             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2520
2521         if (node->child1().useKind() == NotCellUse) {
2522             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2523             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2524
2525             static const double zero = 0;
2526             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2527
2528             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2529             done.append(isNull);
2530
2531             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2532
2533             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2534             static const double one = 1;
2535             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2536             done.append(m_jit.jump());
2537             done.append(isFalse);
2538
2539             isUndefined.link(&m_jit);
2540             static const double NaN = PNaN;
2541             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2542             done.append(m_jit.jump());
2543
2544             isNumber.link(&m_jit);
2545         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2546             typeCheck(
2547                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2548                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2549         }
2550
2551         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2552         done.append(m_jit.jump());
2553     
2554         isInteger.link(&m_jit);
2555         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2556         done.link(&m_jit);
2557 #endif // USE(JSVALUE64)
2558     
2559         doubleResult(resultFPR, node);
2560         return;
2561     }
2562         
2563 #if USE(JSVALUE64)
2564     case Int52RepUse: {
2565         SpeculateStrictInt52Operand value(this, node->child1());
2566         FPRTemporary result(this);
2567         
2568         GPRReg valueGPR = value.gpr();
2569         FPRReg resultFPR = result.fpr();
2570
2571         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2572         
2573         doubleResult(resultFPR, node);
2574         return;
2575     }
2576 #endif // USE(JSVALUE64)
2577         
2578     default:
2579         RELEASE_ASSERT_NOT_REACHED();
2580         return;
2581     }
2582 }
2583
2584 void SpeculativeJIT::compileValueRep(Node* node)
2585 {
2586     switch (node->child1().useKind()) {
2587     case DoubleRepUse: {
2588         SpeculateDoubleOperand value(this, node->child1());
2589         JSValueRegsTemporary result(this);
2590         
2591         FPRReg valueFPR = value.fpr();
2592         JSValueRegs resultRegs = result.regs();
2593         
2594         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2595         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2596         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2597         // local was purified.
2598         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2599             m_jit.purifyNaN(valueFPR);
2600
2601         boxDouble(valueFPR, resultRegs);
2602         
2603         jsValueResult(resultRegs, node);
2604         return;
2605     }
2606         
2607 #if USE(JSVALUE64)
2608     case Int52RepUse: {
2609         SpeculateStrictInt52Operand value(this, node->child1());
2610         GPRTemporary result(this);
2611         
2612         GPRReg valueGPR = value.gpr();
2613         GPRReg resultGPR = result.gpr();
2614         
2615         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2616         
2617         jsValueResult(resultGPR, node);
2618         return;
2619     }
2620 #endif // USE(JSVALUE64)
2621         
2622     default:
2623         RELEASE_ASSERT_NOT_REACHED();
2624         return;
2625     }
2626 }
2627
2628 static double clampDoubleToByte(double d)
2629 {
2630     d += 0.5;
2631     if (!(d > 0))
2632         d = 0;
2633     else if (d > 255)
2634         d = 255;
2635     return d;
2636 }
2637
2638 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2639 {
2640     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2641     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2642     jit.xorPtr(result, result);
2643     MacroAssembler::Jump clamped = jit.jump();
2644     tooBig.link(&jit);
2645     jit.move(JITCompiler::TrustedImm32(255), result);
2646     clamped.link(&jit);
2647     inBounds.link(&jit);
2648 }
2649
2650 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2651 {
2652     // Unordered compare so we pick up NaN
2653     static const double zero = 0;
2654     static const double byteMax = 255;
2655     static const double half = 0.5;
2656     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2657     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2658     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2659     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2660     
2661     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2662     // FIXME: This should probably just use a floating point round!
2663     // https://bugs.webkit.org/show_bug.cgi?id=72054
2664     jit.addDouble(source, scratch);
2665     jit.truncateDoubleToInt32(scratch, result);   
2666     MacroAssembler::Jump truncatedInt = jit.jump();
2667     
2668     tooSmall.link(&jit);
2669     jit.xorPtr(result, result);
2670     MacroAssembler::Jump zeroed = jit.jump();
2671     
2672     tooBig.link(&jit);
2673     jit.move(JITCompiler::TrustedImm32(255), result);
2674     
2675     truncatedInt.link(&jit);
2676     zeroed.link(&jit);
2677
2678 }
2679
2680 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2681 {
2682     if (node->op() == PutByValAlias)
2683         return JITCompiler::Jump();
2684     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2685         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2686     if (view) {
2687         uint32_t length = view->length();
2688         Node* indexNode = m_jit.graph().child(node, 1).node();
2689         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2690             return JITCompiler::Jump();
2691         return m_jit.branch32(
2692             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2693     }
2694     return m_jit.branch32(
2695         MacroAssembler::AboveOrEqual, indexGPR,
2696         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2697 }
2698
2699 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2700 {
2701     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2702     if (!jump.isSet())
2703         return;
2704     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2705 }
2706
2707 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2708 {
2709     JITCompiler::Jump done;
2710     if (outOfBounds.isSet()) {
2711         done = m_jit.jump();
2712         if (node->arrayMode().isInBounds())
2713             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2714         else {
2715             outOfBounds.link(&m_jit);
2716
2717             JITCompiler::Jump notWasteful = m_jit.branch32(
2718                 MacroAssembler::NotEqual,
2719                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2720                 TrustedImm32(WastefulTypedArray));
2721
2722             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2723                 MacroAssembler::Zero,
2724                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2725             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2726             notWasteful.link(&m_jit);
2727         }
2728     }
2729     return done;
2730 }
2731
2732 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2733 {
2734     switch (elementSize(type)) {
2735     case 1:
2736         if (isSigned(type))
2737             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2738         else
2739             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2740         break;
2741     case 2:
2742         if (isSigned(type))
2743             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2744         else
2745             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2746         break;
2747     case 4:
2748         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2749         break;
2750     default:
2751         CRASH();
2752     }
2753 }
2754
2755 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2756 {
2757     if (elementSize(type) < 4 || isSigned(type)) {
2758         int32Result(resultReg, node);
2759         return;
2760     }
2761     
2762     ASSERT(elementSize(type) == 4 && !isSigned(type));
2763     if (node->shouldSpeculateInt32() && canSpeculate) {
2764         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2765         int32Result(resultReg, node);
2766         return;
2767     }
2768     
2769 #if USE(JSVALUE64)
2770     if (node->shouldSpeculateAnyInt()) {
2771         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2772         strictInt52Result(resultReg, node);
2773         return;
2774     }
2775 #endif
2776     
2777     FPRTemporary fresult(this);
2778     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2779     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2780     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2781     positive.link(&m_jit);
2782     doubleResult(fresult.fpr(), node);
2783 }
2784
2785 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2786 {
2787     ASSERT(isInt(type));
2788     
2789     SpeculateCellOperand base(this, node->child1());
2790     SpeculateStrictInt32Operand property(this, node->child2());
2791     StorageOperand storage(this, node->child3());
2792
2793     GPRReg baseReg = base.gpr();
2794     GPRReg propertyReg = property.gpr();
2795     GPRReg storageReg = storage.gpr();
2796
2797     GPRTemporary result(this);
2798     GPRReg resultReg = result.gpr();
2799
2800     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2801
2802     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2803     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2804     bool canSpeculate = true;
2805     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2806 }
2807
2808 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2809     GPRTemporary& value,
2810     GPRReg property,
2811 #if USE(JSVALUE32_64)
2812     GPRTemporary& propertyTag,
2813     GPRTemporary& valueTag,
2814 #endif
2815     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2816 {
2817     bool isAppropriateConstant = false;
2818     if (valueUse->isConstant()) {
2819         JSValue jsValue = valueUse->asJSValue();
2820         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2821         SpeculatedType actualType = speculationFromValue(jsValue);
2822         isAppropriateConstant = (expectedType | actualType) == expectedType;
2823     }
2824     
2825     if (isAppropriateConstant) {
2826         JSValue jsValue = valueUse->asJSValue();
2827         if (!jsValue.isNumber()) {
2828             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2829             return false;
2830         }
2831         double d = jsValue.asNumber();
2832         if (isClamped)
2833             d = clampDoubleToByte(d);
2834         GPRTemporary scratch(this);
2835         GPRReg scratchReg = scratch.gpr();
2836         m_jit.move(Imm32(toInt32(d)), scratchReg);
2837         value.adopt(scratch);
2838     } else {
2839         switch (valueUse.useKind()) {
2840         case Int32Use: {
2841             SpeculateInt32Operand valueOp(this, valueUse);
2842             GPRTemporary scratch(this);
2843             GPRReg scratchReg = scratch.gpr();
2844             m_jit.move(valueOp.gpr(), scratchReg);
2845             if (isClamped)
2846                 compileClampIntegerToByte(m_jit, scratchReg);
2847             value.adopt(scratch);
2848             break;
2849         }
2850             
2851 #if USE(JSVALUE64)
2852         case Int52RepUse: {
2853             SpeculateStrictInt52Operand valueOp(this, valueUse);
2854             GPRTemporary scratch(this);
2855             GPRReg scratchReg = scratch.gpr();
2856             m_jit.move(valueOp.gpr(), scratchReg);
2857             if (isClamped) {
2858                 MacroAssembler::Jump inBounds = m_jit.branch64(
2859                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2860                 MacroAssembler::Jump tooBig = m_jit.branch64(
2861                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2862                 m_jit.move(TrustedImm32(0), scratchReg);
2863                 MacroAssembler::Jump clamped = m_jit.jump();
2864                 tooBig.link(&m_jit);
2865                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2866                 clamped.link(&m_jit);
2867                 inBounds.link(&m_jit);
2868             }
2869             value.adopt(scratch);
2870             break;
2871         }
2872 #endif // USE(JSVALUE64)
2873             
2874         case DoubleRepUse: {
2875             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
2876             if (isClamped) {
2877                 SpeculateDoubleOperand valueOp(this, valueUse);
2878                 GPRTemporary result(this);
2879                 FPRTemporary floatScratch(this);
2880                 FPRReg fpr = valueOp.fpr();
2881                 GPRReg gpr = result.gpr();
2882                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
2883                 value.adopt(result);
2884             } else {
2885 #if USE(JSVALUE32_64)
2886                 GPRTemporary realPropertyTag(this);
2887                 propertyTag.adopt(realPropertyTag);
2888                 GPRReg propertyTagGPR = propertyTag.gpr();
2889
2890                 GPRTemporary realValueTag(this);
2891                 valueTag.adopt(realValueTag);
2892                 GPRReg valueTagGPR = valueTag.gpr();
2893 #endif
2894                 SpeculateDoubleOperand valueOp(this, valueUse);
2895                 GPRTemporary result(this);
2896                 FPRReg fpr = valueOp.fpr();
2897                 GPRReg gpr = result.gpr();
2898                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
2899                 m_jit.xorPtr(gpr, gpr);
2900                 MacroAssembler::JumpList fixed(m_jit.jump());
2901                 notNaN.link(&m_jit);
2902
2903                 fixed.append(m_jit.branchTruncateDoubleToInt32(
2904                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
2905
2906 #if USE(JSVALUE64)
2907                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
2908                 boxDouble(fpr, gpr);
2909 #else
2910                 UNUSED_PARAM(property);
2911                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
2912                 boxDouble(fpr, valueTagGPR, gpr);
2913 #endif
2914                 slowPathCases.append(m_jit.jump());
2915
2916                 fixed.link(&m_jit);
2917                 value.adopt(result);
2918             }
2919             break;
2920         }
2921             
2922         default:
2923             RELEASE_ASSERT_NOT_REACHED();
2924             break;
2925         }
2926     }
2927     return true;
2928 }
2929
2930 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
2931 {
2932     ASSERT(isInt(type));
2933     
2934     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
2935     GPRReg storageReg = storage.gpr();
2936     
2937     Edge valueUse = m_jit.graph().varArgChild(node, 2);
2938     
2939     GPRTemporary value;
2940 #if USE(JSVALUE32_64)
2941     GPRTemporary propertyTag;
2942     GPRTemporary valueTag;
2943 #endif
2944
2945     JITCompiler::JumpList slowPathCases;
2946     
2947     bool result = getIntTypedArrayStoreOperand(
2948         value, property,
2949 #if USE(JSVALUE32_64)
2950         propertyTag, valueTag,
2951 #endif
2952         valueUse, slowPathCases, isClamped(type));
2953     if (!result) {
2954         noResult(node);
2955         return;
2956     }
2957
2958     GPRReg valueGPR = value.gpr();
2959 #if USE(JSVALUE32_64)
2960     GPRReg propertyTagGPR = propertyTag.gpr();
2961     GPRReg valueTagGPR = valueTag.gpr();
2962 #endif
2963
2964     ASSERT_UNUSED(valueGPR, valueGPR != property);
2965     ASSERT(valueGPR != base);
2966     ASSERT(valueGPR != storageReg);
2967     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
2968
2969     switch (elementSize(type)) {
2970     case 1:
2971         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
2972         break;
2973     case 2:
2974         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
2975         break;
2976     case 4:
2977         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
2978         break;
2979     default:
2980         CRASH();
2981     }
2982
2983     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
2984     if (done.isSet())
2985         done.link(&m_jit);
2986
2987     if (!slowPathCases.empty()) {
2988 #if USE(JSVALUE64)
2989         if (node->op() == PutByValDirect) {
2990             addSlowPathGenerator(slowPathCall(
2991                 slowPathCases, this,
2992                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
2993                 NoResult, base, property, valueGPR));
2994         } else {
2995             addSlowPathGenerator(slowPathCall(
2996                 slowPathCases, this,
2997                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
2998                 NoResult, base, property, valueGPR));
2999         }
3000 #else // not USE(JSVALUE64)
3001         if (node->op() == PutByValDirect) {
3002             addSlowPathGenerator(slowPathCall(
3003                 slowPathCases, this,
3004                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3005                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3006         } else {
3007             addSlowPathGenerator(slowPathCall(
3008                 slowPathCases, this,
3009                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3010                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3011         }
3012 #endif
3013     }
3014     
3015     noResult(node);
3016 }
3017
3018 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3019 {
3020     ASSERT(isFloat(type));
3021     
3022     SpeculateCellOperand base(this, node->child1());
3023     SpeculateStrictInt32Operand property(this, node->child2());
3024     StorageOperand storage(this, node->child3());
3025
3026     GPRReg baseReg = base.gpr();
3027     GPRReg propertyReg = property.gpr();
3028     GPRReg storageReg = storage.gpr();
3029
3030     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
3031
3032     FPRTemporary result(this);
3033     FPRReg resultReg = result.fpr();
3034     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3035     switch (elementSize(type)) {
3036     case 4:
3037         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3038         m_jit.convertFloatToDouble(resultReg, resultReg);
3039         break;
3040     case 8: {
3041         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3042         break;
3043     }
3044     default:
3045         RELEASE_ASSERT_NOT_REACHED();
3046     }
3047     
3048     doubleResult(resultReg, node);
3049 }
3050
3051 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3052 {
3053     ASSERT(isFloat(type));
3054     
3055     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3056     GPRReg storageReg = storage.gpr();
3057     
3058     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3059     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3060
3061     SpeculateDoubleOperand valueOp(this, valueUse);
3062     FPRTemporary scratch(this);
3063     FPRReg valueFPR = valueOp.fpr();
3064     FPRReg scratchFPR = scratch.fpr();
3065
3066     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3067     
3068     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3069     
3070     switch (elementSize(type)) {
3071     case 4: {
3072         m_jit.moveDouble(valueFPR, scratchFPR);
3073         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3074         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3075         break;
3076     }
3077     case 8:
3078         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3079         break;
3080     default:
3081         RELEASE_ASSERT_NOT_REACHED();
3082     }
3083
3084     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3085     if (done.isSet())
3086         done.link(&m_jit);
3087     noResult(node);
3088 }
3089
3090 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg)
3091 {
3092     // Check that prototype is an object.
3093     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
3094     
3095     // Initialize scratchReg with the value being checked.
3096     m_jit.move(valueReg, scratchReg);
3097     
3098     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
3099     MacroAssembler::Label loop(&m_jit);
3100     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
3101         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
3102     m_jit.emitLoadStructure(*m_jit.vm(), scratchReg, scratchReg, scratch2Reg);
3103     m_jit.loadPtr(MacroAssembler::Address(scratchReg, Structure::prototypeOffset() + CellPayloadOffset), scratchReg);
3104     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
3105 #if USE(JSVALUE64)
3106     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
3107 #else
3108     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
3109 #endif
3110     
3111     // No match - result is false.
3112 #if USE(JSVALUE64)
3113     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
3114 #else
3115     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
3116 #endif
3117     MacroAssembler::JumpList doneJumps; 
3118     doneJumps.append(m_jit.jump());
3119
3120     performDefaultHasInstance.link(&m_jit);
3121     silentSpillAllRegisters(scratchReg);
3122     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg); 
3123     silentFillAllRegisters();
3124     m_jit.exceptionCheck();
3125 #if USE(JSVALUE64)
3126     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
3127 #endif
3128     doneJumps.append(m_jit.jump());
3129     
3130     isInstance.link(&m_jit);
3131 #if USE(JSVALUE64)
3132     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), scratchReg);
3133 #else
3134     m_jit.move(MacroAssembler::TrustedImm32(1), scratchReg);
3135 #endif
3136     
3137     doneJumps.link(&m_jit);
3138 }
3139
3140 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3141 {
3142     SpeculateCellOperand base(this, node->child1());
3143
3144     GPRReg baseGPR = base.gpr();
3145
3146     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3147
3148     noResult(node);
3149 }
3150
3151 void SpeculativeJIT::compileParseInt(Node* node)
3152 {
3153     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3154
3155     GPRFlushedCallResult resultPayload(this);
3156     GPRReg resultPayloadGPR = resultPayload.gpr();
3157 #if USE(JSVALUE64)
3158     JSValueRegs resultRegs(resultPayloadGPR);
3159 #else
3160     GPRFlushedCallResult2 resultTag(this);
3161     GPRReg resultTagGPR = resultTag.gpr();
3162     JSValueRegs resultRegs(resultTagGPR, resultPayloadGPR);
3163 #endif
3164
3165     if (node->child2()) {
3166         SpeculateInt32Operand radix(this, node->child2());
3167         GPRReg radixGPR = radix.gpr();
3168         if (node->child1().useKind() == UntypedUse) {
3169             JSValueOperand value(this, node->child1());
3170 #if USE(JSVALUE64)
3171             auto result = resultRegs.gpr();
3172             auto valueReg = value.gpr();
3173 #else
3174             auto result = resultRegs;
3175             auto valueReg = value.jsValueRegs();
3176 #endif
3177
3178             flushRegisters();
3179             callOperation(operationParseIntGeneric, result, valueReg, radixGPR);
3180             m_jit.exceptionCheck();
3181         } else {
3182             SpeculateCellOperand value(this, node->child1());
3183             GPRReg valueGPR = value.gpr();
3184             speculateString(node->child1(), valueGPR);
3185
3186 #if USE(JSVALUE64)
3187             auto result = resultRegs.gpr();
3188 #else
3189             auto result = resultRegs;
3190 #endif
3191
3192             flushRegisters();
3193             callOperation(operationParseIntString, result, valueGPR, radixGPR);
3194             m_jit.exceptionCheck();
3195         }
3196     } else {
3197         if (node->child1().useKind() == UntypedUse) {
3198             JSValueOperand value(this, node->child1());
3199 #if USE(JSVALUE64)
3200             auto result = resultRegs.gpr();
3201 #else
3202             auto result = resultRegs;
3203 #endif
3204             JSValueRegs valueRegs = value.jsValueRegs();
3205
3206             flushRegisters();
3207             callOperation(operationParseIntNoRadixGeneric, result, valueRegs);
3208             m_jit.exceptionCheck();
3209         } else {
3210             SpeculateCellOperand value(this, node->child1());
3211             GPRReg valueGPR = value.gpr();
3212             speculateString(node->child1(), valueGPR);
3213
3214             flushRegisters();
3215             callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3216             m_jit.exceptionCheck();
3217         }
3218     }
3219
3220     jsValueResult(resultRegs, node);
3221 }
3222
3223 void SpeculativeJIT::compileInstanceOf(Node* node)
3224 {
3225     if (node->child1().useKind() == UntypedUse) {
3226         // It might not be a cell. Speculate less aggressively.
3227         // Or: it might only be used once (i.e. by us), so we get zero benefit
3228         // from speculating any more aggressively than we absolutely need to.
3229         
3230         JSValueOperand value(this, node->child1());
3231         SpeculateCellOperand prototype(this, node->child2());
3232         GPRTemporary scratch(this);
3233         GPRTemporary scratch2(this);
3234         
3235         GPRReg prototypeReg = prototype.gpr();
3236         GPRReg scratchReg = scratch.gpr();
3237         GPRReg scratch2Reg = scratch2.gpr();
3238         
3239         MacroAssembler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs());
3240         GPRReg valueReg = value.jsValueRegs().payloadGPR();
3241         moveFalseTo(scratchReg);
3242
3243         MacroAssembler::Jump done = m_jit.jump();
3244         
3245         isCell.link(&m_jit);
3246         
3247         compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg);
3248         
3249         done.link(&m_jit);
3250
3251         blessedBooleanResult(scratchReg, node);
3252         return;
3253     }
3254     
3255     SpeculateCellOperand value(this, node->child1());
3256     SpeculateCellOperand prototype(this, node->child2());
3257     
3258     GPRTemporary scratch(this);
3259     GPRTemporary scratch2(this);
3260     
3261     GPRReg valueReg = value.gpr();
3262     GPRReg prototypeReg = prototype.gpr();
3263     GPRReg scratchReg = scratch.gpr();
3264     GPRReg scratch2Reg = scratch2.gpr();
3265     
3266     compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg);
3267
3268     blessedBooleanResult(scratchReg, node);
3269 }
3270
3271 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3272 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3273 {
3274     Edge& leftChild = node->child1();
3275     Edge& rightChild = node->child2();
3276
3277     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3278         JSValueOperand left(this, leftChild);
3279         JSValueOperand right(this, rightChild);
3280         JSValueRegs leftRegs = left.jsValueRegs();
3281         JSValueRegs rightRegs = right.jsValueRegs();
3282 #if USE(JSVALUE64)
3283         GPRTemporary result(this);
3284         JSValueRegs resultRegs = JSValueRegs(result.gpr());
3285 #else
3286         GPRTemporary resultTag(this);
3287         GPRTemporary resultPayload(this);
3288         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3289 #endif
3290         flushRegisters();
3291         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3292         m_jit.exceptionCheck();
3293
3294         jsValueResult(resultRegs, node);
3295         return;
3296     }
3297
3298     std::optional<JSValueOperand> left;
3299     std::optional<JSValueOperand> right;
3300
3301     JSValueRegs leftRegs;
3302     JSValueRegs rightRegs;
3303
3304 #if USE(JSVALUE64)
3305     GPRTemporary result(this);
3306     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3307     GPRTemporary scratch(this);
3308     GPRReg scratchGPR = scratch.gpr();
3309 #else
3310     GPRTemporary resultTag(this);
3311     GPRTemporary resultPayload(this);
3312     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3313     GPRReg scratchGPR = resultTag.gpr();
3314 #endif
3315
3316     SnippetOperand leftOperand;
3317     SnippetOperand rightOperand;
3318
3319     // The snippet generator does not support both operands being constant. If the left
3320     // operand is already const, we'll ignore the right operand's constness.
3321     if (leftChild->isInt32Constant())
3322         leftOperand.setConstInt32(leftChild->asInt32());
3323     else if (rightChild->isInt32Constant())
3324         rightOperand.setConstInt32(rightChild->asInt32());
3325
3326     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3327
3328     if (!leftOperand.isConst()) {
3329         left.emplace(this, leftChild);
3330         leftRegs = left->jsValueRegs();
3331     }
3332     if (!rightOperand.isConst()) {
3333         right.emplace(this, rightChild);
3334         rightRegs = right->jsValueRegs();
3335     }
3336
3337     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3338     gen.generateFastPath(m_jit);
3339
3340     ASSERT(gen.didEmitFastPath());
3341     gen.endJumpList().append(m_jit.jump());
3342
3343     gen.slowPathJumpList().link(&m_jit);
3344     silentSpillAllRegisters(resultRegs);
3345
3346     if (leftOperand.isConst()) {
3347         leftRegs = resultRegs;
3348         m_jit.moveValue(leftChild->asJSValue(), leftRegs);