[FTL] Drop NewRegexp for String.prototype.match with RegExp + global flag
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCapabilities.h"
37 #include "DFGMayExit.h"
38 #include "DFGOSRExitFuzz.h"
39 #include "DFGSaneStringGetByValSlowPathGenerator.h"
40 #include "DFGSlowPathGenerator.h"
41 #include "DFGSnippetParams.h"
42 #include "DirectArguments.h"
43 #include "JITAddGenerator.h"
44 #include "JITBitAndGenerator.h"
45 #include "JITBitOrGenerator.h"
46 #include "JITBitXorGenerator.h"
47 #include "JITDivGenerator.h"
48 #include "JITLeftShiftGenerator.h"
49 #include "JITMulGenerator.h"
50 #include "JITRightShiftGenerator.h"
51 #include "JITSubGenerator.h"
52 #include "JSAsyncFunction.h"
53 #include "JSAsyncGeneratorFunction.h"
54 #include "JSCInlines.h"
55 #include "JSFixedArray.h"
56 #include "JSGeneratorFunction.h"
57 #include "JSLexicalEnvironment.h"
58 #include "JSPropertyNameEnumerator.h"
59 #include "LinkBuffer.h"
60 #include "RegExpConstructor.h"
61 #include "ScopedArguments.h"
62 #include "ScratchRegisterAllocator.h"
63 #include "SuperSampler.h"
64 #include "WeakMapImpl.h"
65 #include <wtf/BitVector.h>
66 #include <wtf/Box.h>
67 #include <wtf/MathExtras.h>
68
69 namespace JSC { namespace DFG {
70
71 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
72     : m_compileOkay(true)
73     , m_jit(jit)
74     , m_graph(m_jit.graph())
75     , m_currentNode(0)
76     , m_lastGeneratedNode(LastNodeType)
77     , m_indexInBlock(0)
78     , m_indexMaskingMode(Options::enableSpectreMitigations() ? IndexMaskingEnabled : IndexMaskingDisabled)
79     , m_generationInfo(m_jit.graph().frameRegisterCount())
80     , m_state(m_jit.graph())
81     , m_interpreter(m_jit.graph(), m_state)
82     , m_stream(&jit.jitCode()->variableEventStream)
83     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
84 {
85 }
86
87 SpeculativeJIT::~SpeculativeJIT()
88 {
89 }
90
91 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
92 {
93     IndexingType indexingType = structure->indexingType();
94     bool hasIndexingHeader = hasIndexedProperties(indexingType);
95
96     unsigned inlineCapacity = structure->inlineCapacity();
97     unsigned outOfLineCapacity = structure->outOfLineCapacity();
98     
99     GPRTemporary scratch(this);
100     GPRTemporary scratch2(this);
101     GPRReg scratchGPR = scratch.gpr();
102     GPRReg scratch2GPR = scratch2.gpr();
103
104     ASSERT(vectorLength >= numElements);
105     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
106     
107     JITCompiler::JumpList slowCases;
108
109     size_t size = 0;
110     if (hasIndexingHeader)
111         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
112     size += outOfLineCapacity * sizeof(JSValue);
113
114     m_jit.move(TrustedImmPtr(nullptr), storageGPR);
115
116     if (size) {
117         if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
118             m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
119             
120             m_jit.addPtr(
121                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
122                 storageGPR);
123             
124             if (hasIndexingHeader)
125                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
126         } else
127             slowCases.append(m_jit.jump());
128     }
129
130     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
131     Allocator allocator = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
132     if (allocator) {
133         uint32_t mask = WTF::computeIndexingMask(vectorLength);
134         emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, TrustedImm32(mask), scratch2GPR, slowCases);
135         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
136     } else
137         slowCases.append(m_jit.jump());
138
139     // I want a slow path that also loads out the storage pointer, and that's
140     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
141     // of work for a very small piece of functionality. :-/
142     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
143         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
144         structure, vectorLength));
145
146     if (numElements < vectorLength) {
147 #if USE(JSVALUE64)
148         if (hasDouble(structure->indexingType()))
149             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
150         else
151             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
152         for (unsigned i = numElements; i < vectorLength; ++i)
153             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
154 #else
155         EncodedValueDescriptor value;
156         if (hasDouble(structure->indexingType()))
157             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
158         else
159             value.asInt64 = JSValue::encode(JSValue());
160         for (unsigned i = numElements; i < vectorLength; ++i) {
161             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
162             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
163         }
164 #endif
165     }
166     
167     if (hasIndexingHeader)
168         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
169     
170     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
171     
172     m_jit.mutatorFence(*m_jit.vm());
173 }
174
175 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
176 {
177     if (inlineCallFrame && !inlineCallFrame->isVarargs())
178         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
179     else {
180         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
181         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
182         if (!includeThis)
183             m_jit.sub32(TrustedImm32(1), lengthGPR);
184     }
185 }
186
187 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
188 {
189     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
190 }
191
192 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
193 {
194     if (origin.inlineCallFrame) {
195         if (origin.inlineCallFrame->isClosureCall) {
196             m_jit.loadPtr(
197                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
198                 calleeGPR);
199         } else {
200             m_jit.move(
201                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
202                 calleeGPR);
203         }
204     } else
205         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
206 }
207
208 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
209 {
210     m_jit.addPtr(
211         TrustedImm32(
212             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
213         GPRInfo::callFrameRegister, startGPR);
214 }
215
216 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
217 {
218     if (!Options::useOSRExitFuzz()
219         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
220         || !doOSRExitFuzzing())
221         return MacroAssembler::Jump();
222     
223     MacroAssembler::Jump result;
224     
225     m_jit.pushToSave(GPRInfo::regT0);
226     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
227     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
228     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
229     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
230     unsigned at = Options::fireOSRExitFuzzAt();
231     if (at || atOrAfter) {
232         unsigned threshold;
233         MacroAssembler::RelationalCondition condition;
234         if (atOrAfter) {
235             threshold = atOrAfter;
236             condition = MacroAssembler::Below;
237         } else {
238             threshold = at;
239             condition = MacroAssembler::NotEqual;
240         }
241         MacroAssembler::Jump ok = m_jit.branch32(
242             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
243         m_jit.popToRestore(GPRInfo::regT0);
244         result = m_jit.jump();
245         ok.link(&m_jit);
246     }
247     m_jit.popToRestore(GPRInfo::regT0);
248     
249     return result;
250 }
251
252 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
253 {
254     if (!m_compileOkay)
255         return;
256     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
257     if (fuzzJump.isSet()) {
258         JITCompiler::JumpList jumpsToFail;
259         jumpsToFail.append(fuzzJump);
260         jumpsToFail.append(jumpToFail);
261         m_jit.appendExitInfo(jumpsToFail);
262     } else
263         m_jit.appendExitInfo(jumpToFail);
264     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
265 }
266
267 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
268 {
269     if (!m_compileOkay)
270         return;
271     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
272     if (fuzzJump.isSet()) {
273         JITCompiler::JumpList myJumpsToFail;
274         myJumpsToFail.append(jumpsToFail);
275         myJumpsToFail.append(fuzzJump);
276         m_jit.appendExitInfo(myJumpsToFail);
277     } else
278         m_jit.appendExitInfo(jumpsToFail);
279     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
280 }
281
282 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
283 {
284     if (!m_compileOkay)
285         return OSRExitJumpPlaceholder();
286     unsigned index = m_jit.jitCode()->osrExit.size();
287     m_jit.appendExitInfo();
288     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
289     return OSRExitJumpPlaceholder(index);
290 }
291
292 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
293 {
294     return speculationCheck(kind, jsValueSource, nodeUse.node());
295 }
296
297 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
298 {
299     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
300 }
301
302 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
303 {
304     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
305 }
306
307 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
308 {
309     if (!m_compileOkay)
310         return;
311     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
312     m_jit.appendExitInfo(jumpToFail);
313     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
314 }
315
316 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
317 {
318     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
319 }
320
321 void SpeculativeJIT::emitInvalidationPoint(Node* node)
322 {
323     if (!m_compileOkay)
324         return;
325     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
326     m_jit.jitCode()->appendOSRExit(OSRExit(
327         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
328         this, m_stream->size()));
329     info.m_replacementSource = m_jit.watchpointLabel();
330     ASSERT(info.m_replacementSource.isSet());
331     noResult(node);
332 }
333
334 void SpeculativeJIT::unreachable(Node* node)
335 {
336     m_compileOkay = false;
337     m_jit.abortWithReason(DFGUnreachableNode, node->op());
338 }
339
340 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
341 {
342     if (!m_compileOkay)
343         return;
344     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
345     m_compileOkay = false;
346     if (verboseCompilationEnabled())
347         dataLog("Bailing compilation.\n");
348 }
349
350 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
351 {
352     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
353 }
354
355 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
356 {
357     ASSERT(needsTypeCheck(edge, typesPassedThrough));
358     m_interpreter.filter(edge, typesPassedThrough);
359     speculationCheck(exitKind, source, edge.node(), jumpToFail);
360 }
361
362 RegisterSet SpeculativeJIT::usedRegisters()
363 {
364     RegisterSet result;
365     
366     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
367         GPRReg gpr = GPRInfo::toRegister(i);
368         if (m_gprs.isInUse(gpr))
369             result.set(gpr);
370     }
371     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
372         FPRReg fpr = FPRInfo::toRegister(i);
373         if (m_fprs.isInUse(fpr))
374             result.set(fpr);
375     }
376     
377     result.merge(RegisterSet::stubUnavailableRegisters());
378     
379     return result;
380 }
381
382 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
383 {
384     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
385 }
386
387 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
388 {
389     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
390 }
391
392 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
393 {
394     for (auto& slowPathGenerator : m_slowPathGenerators) {
395         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
396         slowPathGenerator->generate(this);
397     }
398     for (auto& slowPathLambda : m_slowPathLambdas) {
399         Node* currentNode = slowPathLambda.currentNode;
400         m_currentNode = currentNode;
401         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
402         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
403         slowPathLambda.generator();
404         m_outOfLineStreamIndex = std::nullopt;
405     }
406 }
407
408 void SpeculativeJIT::clearGenerationInfo()
409 {
410     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
411         m_generationInfo[i] = GenerationInfo();
412     m_gprs = RegisterBank<GPRInfo>();
413     m_fprs = RegisterBank<FPRInfo>();
414 }
415
416 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
417 {
418     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
419     Node* node = info.node();
420     DataFormat registerFormat = info.registerFormat();
421     ASSERT(registerFormat != DataFormatNone);
422     ASSERT(registerFormat != DataFormatDouble);
423         
424     SilentSpillAction spillAction;
425     SilentFillAction fillAction;
426         
427     if (!info.needsSpill())
428         spillAction = DoNothingForSpill;
429     else {
430 #if USE(JSVALUE64)
431         ASSERT(info.gpr() == source);
432         if (registerFormat == DataFormatInt32)
433             spillAction = Store32Payload;
434         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
435             spillAction = StorePtr;
436         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
437             spillAction = Store64;
438         else {
439             ASSERT(registerFormat & DataFormatJS);
440             spillAction = Store64;
441         }
442 #elif USE(JSVALUE32_64)
443         if (registerFormat & DataFormatJS) {
444             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
445             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
446         } else {
447             ASSERT(info.gpr() == source);
448             spillAction = Store32Payload;
449         }
450 #endif
451     }
452         
453     if (registerFormat == DataFormatInt32) {
454         ASSERT(info.gpr() == source);
455         ASSERT(isJSInt32(info.registerFormat()));
456         if (node->hasConstant()) {
457             ASSERT(node->isInt32Constant());
458             fillAction = SetInt32Constant;
459         } else
460             fillAction = Load32Payload;
461     } else if (registerFormat == DataFormatBoolean) {
462 #if USE(JSVALUE64)
463         RELEASE_ASSERT_NOT_REACHED();
464 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
465         fillAction = DoNothingForFill;
466 #endif
467 #elif USE(JSVALUE32_64)
468         ASSERT(info.gpr() == source);
469         if (node->hasConstant()) {
470             ASSERT(node->isBooleanConstant());
471             fillAction = SetBooleanConstant;
472         } else
473             fillAction = Load32Payload;
474 #endif
475     } else if (registerFormat == DataFormatCell) {
476         ASSERT(info.gpr() == source);
477         if (node->hasConstant()) {
478             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
479             node->asCell(); // To get the assertion.
480             fillAction = SetCellConstant;
481         } else {
482 #if USE(JSVALUE64)
483             fillAction = LoadPtr;
484 #else
485             fillAction = Load32Payload;
486 #endif
487         }
488     } else if (registerFormat == DataFormatStorage) {
489         ASSERT(info.gpr() == source);
490         fillAction = LoadPtr;
491     } else if (registerFormat == DataFormatInt52) {
492         if (node->hasConstant())
493             fillAction = SetInt52Constant;
494         else if (info.spillFormat() == DataFormatInt52)
495             fillAction = Load64;
496         else if (info.spillFormat() == DataFormatStrictInt52)
497             fillAction = Load64ShiftInt52Left;
498         else if (info.spillFormat() == DataFormatNone)
499             fillAction = Load64;
500         else {
501             RELEASE_ASSERT_NOT_REACHED();
502 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
503             fillAction = Load64; // Make GCC happy.
504 #endif
505         }
506     } else if (registerFormat == DataFormatStrictInt52) {
507         if (node->hasConstant())
508             fillAction = SetStrictInt52Constant;
509         else if (info.spillFormat() == DataFormatInt52)
510             fillAction = Load64ShiftInt52Right;
511         else if (info.spillFormat() == DataFormatStrictInt52)
512             fillAction = Load64;
513         else if (info.spillFormat() == DataFormatNone)
514             fillAction = Load64;
515         else {
516             RELEASE_ASSERT_NOT_REACHED();
517 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
518             fillAction = Load64; // Make GCC happy.
519 #endif
520         }
521     } else {
522         ASSERT(registerFormat & DataFormatJS);
523 #if USE(JSVALUE64)
524         ASSERT(info.gpr() == source);
525         if (node->hasConstant()) {
526             if (node->isCellConstant())
527                 fillAction = SetTrustedJSConstant;
528             else
529                 fillAction = SetJSConstant;
530         } else if (info.spillFormat() == DataFormatInt32) {
531             ASSERT(registerFormat == DataFormatJSInt32);
532             fillAction = Load32PayloadBoxInt;
533         } else
534             fillAction = Load64;
535 #else
536         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
537         if (node->hasConstant())
538             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
539         else if (info.payloadGPR() == source)
540             fillAction = Load32Payload;
541         else { // Fill the Tag
542             switch (info.spillFormat()) {
543             case DataFormatInt32:
544                 ASSERT(registerFormat == DataFormatJSInt32);
545                 fillAction = SetInt32Tag;
546                 break;
547             case DataFormatCell:
548                 ASSERT(registerFormat == DataFormatJSCell);
549                 fillAction = SetCellTag;
550                 break;
551             case DataFormatBoolean:
552                 ASSERT(registerFormat == DataFormatJSBoolean);
553                 fillAction = SetBooleanTag;
554                 break;
555             default:
556                 fillAction = Load32Tag;
557                 break;
558             }
559         }
560 #endif
561     }
562         
563     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
564 }
565     
566 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
567 {
568     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
569     Node* node = info.node();
570     ASSERT(info.registerFormat() == DataFormatDouble);
571
572     SilentSpillAction spillAction;
573     SilentFillAction fillAction;
574         
575     if (!info.needsSpill())
576         spillAction = DoNothingForSpill;
577     else {
578         ASSERT(!node->hasConstant());
579         ASSERT(info.spillFormat() == DataFormatNone);
580         ASSERT(info.fpr() == source);
581         spillAction = StoreDouble;
582     }
583         
584 #if USE(JSVALUE64)
585     if (node->hasConstant()) {
586         node->asNumber(); // To get the assertion.
587         fillAction = SetDoubleConstant;
588     } else {
589         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
590         fillAction = LoadDouble;
591     }
592 #elif USE(JSVALUE32_64)
593     ASSERT(info.registerFormat() == DataFormatDouble);
594     if (node->hasConstant()) {
595         node->asNumber(); // To get the assertion.
596         fillAction = SetDoubleConstant;
597     } else
598         fillAction = LoadDouble;
599 #endif
600
601     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
602 }
603     
604 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
605 {
606     switch (plan.spillAction()) {
607     case DoNothingForSpill:
608         break;
609     case Store32Tag:
610         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
611         break;
612     case Store32Payload:
613         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
614         break;
615     case StorePtr:
616         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
617         break;
618 #if USE(JSVALUE64)
619     case Store64:
620         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
621         break;
622 #endif
623     case StoreDouble:
624         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
625         break;
626     default:
627         RELEASE_ASSERT_NOT_REACHED();
628     }
629 }
630     
631 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
632 {
633     switch (plan.fillAction()) {
634     case DoNothingForFill:
635         break;
636     case SetInt32Constant:
637         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
638         break;
639 #if USE(JSVALUE64)
640     case SetInt52Constant:
641         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
642         break;
643     case SetStrictInt52Constant:
644         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
645         break;
646 #endif // USE(JSVALUE64)
647     case SetBooleanConstant:
648         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
649         break;
650     case SetCellConstant:
651         ASSERT(plan.node()->constant()->value().isCell());
652         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
653         break;
654 #if USE(JSVALUE64)
655     case SetTrustedJSConstant:
656         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
657         break;
658     case SetJSConstant:
659         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
660         break;
661     case SetDoubleConstant:
662         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
663         break;
664     case Load32PayloadBoxInt:
665         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
666         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
667         break;
668     case Load32PayloadConvertToInt52:
669         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
670         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
671         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
672         break;
673     case Load32PayloadSignExtend:
674         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
675         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
676         break;
677 #else
678     case SetJSConstantTag:
679         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
680         break;
681     case SetJSConstantPayload:
682         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
683         break;
684     case SetInt32Tag:
685         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
686         break;
687     case SetCellTag:
688         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
689         break;
690     case SetBooleanTag:
691         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
692         break;
693     case SetDoubleConstant:
694         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
695         break;
696 #endif
697     case Load32Tag:
698         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
699         break;
700     case Load32Payload:
701         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
702         break;
703     case LoadPtr:
704         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
705         break;
706 #if USE(JSVALUE64)
707     case Load64:
708         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
709         break;
710     case Load64ShiftInt52Right:
711         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
712         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
713         break;
714     case Load64ShiftInt52Left:
715         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
716         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
717         break;
718 #endif
719     case LoadDouble:
720         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
721         break;
722     default:
723         RELEASE_ASSERT_NOT_REACHED();
724     }
725 }
726
727 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
728 {
729     JITCompiler::JumpList result;
730     
731     switch (arrayMode.type()) {
732     case Array::Int32:
733     case Array::Double:
734     case Array::Contiguous:
735     case Array::Undecided:
736     case Array::ArrayStorage: {
737         IndexingType shape = arrayMode.shapeMask();
738         switch (arrayMode.arrayClass()) {
739         case Array::OriginalArray:
740             RELEASE_ASSERT_NOT_REACHED();
741             return result;
742
743         case Array::Array:
744             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
745             result.append(m_jit.branch32(
746                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
747             return result;
748
749         case Array::NonArray:
750         case Array::OriginalNonArray:
751             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
752             result.append(m_jit.branch32(
753                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
754             return result;
755
756         case Array::PossiblyArray:
757             m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
758             result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
759             return result;
760         }
761
762         RELEASE_ASSERT_NOT_REACHED();
763         return result;
764     }
765
766     case Array::SlowPutArrayStorage: {
767         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
768
769         switch (arrayMode.arrayClass()) {
770         case Array::OriginalArray:
771             RELEASE_ASSERT_NOT_REACHED();
772             return result;
773
774         case Array::Array:
775             result.append(
776                 m_jit.branchTest32(
777                     MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
778             break;
779
780         case Array::NonArray:
781         case Array::OriginalNonArray:
782             result.append(
783                 m_jit.branchTest32(
784                     MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
785             break;
786
787         case Array::PossiblyArray:
788             break;
789         }
790
791         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
792         m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
793         result.append(
794             m_jit.branch32(
795                 MacroAssembler::Above, tempGPR,
796                 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
797         return result;
798     }
799     default:
800         CRASH();
801         break;
802     }
803     
804     return result;
805 }
806
807 void SpeculativeJIT::checkArray(Node* node)
808 {
809     ASSERT(node->arrayMode().isSpecific());
810     ASSERT(!node->arrayMode().doesConversion());
811     
812     SpeculateCellOperand base(this, node->child1());
813     GPRReg baseReg = base.gpr();
814     
815     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
816         noResult(m_currentNode);
817         return;
818     }
819     
820     switch (node->arrayMode().type()) {
821     case Array::AnyTypedArray:
822     case Array::String:
823         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
824         return;
825     case Array::Int32:
826     case Array::Double:
827     case Array::Contiguous:
828     case Array::Undecided:
829     case Array::ArrayStorage:
830     case Array::SlowPutArrayStorage: {
831         GPRTemporary temp(this);
832         GPRReg tempGPR = temp.gpr();
833         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
834         speculationCheck(
835             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
836             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
837         
838         noResult(m_currentNode);
839         return;
840     }
841     case Array::DirectArguments:
842         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
843         noResult(m_currentNode);
844         return;
845     case Array::ScopedArguments:
846         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
847         noResult(m_currentNode);
848         return;
849     default:
850         speculateCellTypeWithoutTypeFiltering(
851             node->child1(), baseReg,
852             typeForTypedArrayType(node->arrayMode().typedArrayType()));
853         noResult(m_currentNode);
854         return;
855     }
856 }
857
858 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
859 {
860     ASSERT(node->arrayMode().doesConversion());
861     
862     GPRTemporary temp(this);
863     GPRTemporary structure;
864     GPRReg tempGPR = temp.gpr();
865     GPRReg structureGPR = InvalidGPRReg;
866     
867     if (node->op() != ArrayifyToStructure) {
868         GPRTemporary realStructure(this);
869         structure.adopt(realStructure);
870         structureGPR = structure.gpr();
871     }
872         
873     // We can skip all that comes next if we already have array storage.
874     MacroAssembler::JumpList slowPath;
875     
876     if (node->op() == ArrayifyToStructure) {
877         slowPath.append(m_jit.branchWeakStructure(
878             JITCompiler::NotEqual,
879             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
880             node->structure()));
881     } else {
882         m_jit.load8(
883             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
884         
885         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
886     }
887     
888     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
889         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
890     
891     noResult(m_currentNode);
892 }
893
894 void SpeculativeJIT::arrayify(Node* node)
895 {
896     ASSERT(node->arrayMode().isSpecific());
897     
898     SpeculateCellOperand base(this, node->child1());
899     
900     if (!node->child2()) {
901         arrayify(node, base.gpr(), InvalidGPRReg);
902         return;
903     }
904     
905     SpeculateInt32Operand property(this, node->child2());
906     
907     arrayify(node, base.gpr(), property.gpr());
908 }
909
910 GPRReg SpeculativeJIT::fillStorage(Edge edge)
911 {
912     VirtualRegister virtualRegister = edge->virtualRegister();
913     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
914     
915     switch (info.registerFormat()) {
916     case DataFormatNone: {
917         if (info.spillFormat() == DataFormatStorage) {
918             GPRReg gpr = allocate();
919             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
920             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
921             info.fillStorage(*m_stream, gpr);
922             return gpr;
923         }
924         
925         // Must be a cell; fill it as a cell and then return the pointer.
926         return fillSpeculateCell(edge);
927     }
928         
929     case DataFormatStorage: {
930         GPRReg gpr = info.gpr();
931         m_gprs.lock(gpr);
932         return gpr;
933     }
934         
935     default:
936         return fillSpeculateCell(edge);
937     }
938 }
939
940 void SpeculativeJIT::useChildren(Node* node)
941 {
942     if (node->flags() & NodeHasVarArgs) {
943         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
944             if (!!m_jit.graph().m_varArgChildren[childIdx])
945                 use(m_jit.graph().m_varArgChildren[childIdx]);
946         }
947     } else {
948         Edge child1 = node->child1();
949         if (!child1) {
950             ASSERT(!node->child2() && !node->child3());
951             return;
952         }
953         use(child1);
954         
955         Edge child2 = node->child2();
956         if (!child2) {
957             ASSERT(!node->child3());
958             return;
959         }
960         use(child2);
961         
962         Edge child3 = node->child3();
963         if (!child3)
964             return;
965         use(child3);
966     }
967 }
968
969 void SpeculativeJIT::compileTryGetById(Node* node)
970 {
971     switch (node->child1().useKind()) {
972     case CellUse: {
973         SpeculateCellOperand base(this, node->child1());
974         JSValueRegsTemporary result(this, Reuse, base);
975
976         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
977         JSValueRegs resultRegs = result.regs();
978
979         base.use();
980
981         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, AccessType::TryGet);
982
983         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
984         break;
985     }
986
987     case UntypedUse: {
988         JSValueOperand base(this, node->child1());
989         JSValueRegsTemporary result(this, Reuse, base);
990
991         JSValueRegs baseRegs = base.jsValueRegs();
992         JSValueRegs resultRegs = result.regs();
993
994         base.use();
995
996         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
997
998         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, AccessType::TryGet);
999
1000         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1001         break;
1002     }
1003
1004     default:
1005         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1006         break;
1007     } 
1008 }
1009
1010 void SpeculativeJIT::compileIn(Node* node)
1011 {
1012     SpeculateCellOperand base(this, node->child1());
1013     GPRReg baseGPR = base.gpr();
1014     
1015     if (JSString* string = node->child2()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
1016         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1017             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1018             
1019             GPRTemporary result(this);
1020             GPRReg resultGPR = result.gpr();
1021
1022             use(node->child2());
1023             
1024             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1025             MacroAssembler::Label done = m_jit.label();
1026             
1027             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1028             // we can cast it to const AtomicStringImpl* safely.
1029             auto slowPath = slowPathCall(
1030                 jump.m_jump, this, operationInOptimize,
1031                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1032                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1033             
1034             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1035             stubInfo->codeOrigin = node->origin.semantic;
1036             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1037             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1038             stubInfo->patch.thisGPR = static_cast<int8_t>(InvalidGPRReg);
1039 #if USE(JSVALUE32_64)
1040             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1041             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1042             stubInfo->patch.thisTagGPR = static_cast<int8_t>(InvalidGPRReg);
1043 #endif
1044             stubInfo->patch.usedRegisters = usedRegisters();
1045
1046             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1047             addSlowPathGenerator(WTFMove(slowPath));
1048
1049             base.use();
1050
1051             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1052             return;
1053         }
1054     }
1055
1056     JSValueOperand key(this, node->child2());
1057     JSValueRegs regs = key.jsValueRegs();
1058         
1059     GPRFlushedCallResult result(this);
1060     GPRReg resultGPR = result.gpr();
1061         
1062     base.use();
1063     key.use();
1064         
1065     flushRegisters();
1066     callOperation(
1067         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1068         baseGPR, regs);
1069     m_jit.exceptionCheck();
1070     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1071 }
1072
1073 void SpeculativeJIT::compileDeleteById(Node* node)
1074 {
1075     JSValueOperand value(this, node->child1());
1076     GPRFlushedCallResult result(this);
1077
1078     JSValueRegs valueRegs = value.jsValueRegs();
1079     GPRReg resultGPR = result.gpr();
1080
1081     value.use();
1082
1083     flushRegisters();
1084     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1085     m_jit.exceptionCheck();
1086
1087     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1088 }
1089
1090 void SpeculativeJIT::compileDeleteByVal(Node* node)
1091 {
1092     JSValueOperand base(this, node->child1());
1093     JSValueOperand key(this, node->child2());
1094     GPRFlushedCallResult result(this);
1095
1096     JSValueRegs baseRegs = base.jsValueRegs();
1097     JSValueRegs keyRegs = key.jsValueRegs();
1098     GPRReg resultGPR = result.gpr();
1099
1100     base.use();
1101     key.use();
1102
1103     flushRegisters();
1104     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1105     m_jit.exceptionCheck();
1106
1107     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1108 }
1109
1110 void SpeculativeJIT::compilePushWithScope(Node* node)
1111 {
1112     SpeculateCellOperand currentScope(this, node->child1());
1113     GPRReg currentScopeGPR = currentScope.gpr();
1114
1115     GPRFlushedCallResult result(this);
1116     GPRReg resultGPR = result.gpr();
1117
1118     auto objectEdge = node->child2();
1119     if (objectEdge.useKind() == ObjectUse) {
1120         SpeculateCellOperand object(this, objectEdge);
1121         GPRReg objectGPR = object.gpr();
1122         speculateObject(objectEdge, objectGPR);
1123
1124         flushRegisters();
1125         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1126         // No exception check here as we did not have to call toObject().
1127     } else {
1128         ASSERT(objectEdge.useKind() == UntypedUse);
1129         JSValueOperand object(this, objectEdge);
1130         JSValueRegs objectRegs = object.jsValueRegs();
1131
1132         flushRegisters();
1133         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1134         m_jit.exceptionCheck();
1135     }
1136     
1137     cellResult(resultGPR, node);
1138 }
1139
1140 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1141 {
1142     unsigned branchIndexInBlock = detectPeepHoleBranch();
1143     if (branchIndexInBlock != UINT_MAX) {
1144         Node* branchNode = m_block->at(branchIndexInBlock);
1145
1146         ASSERT(node->adjustedRefCount() == 1);
1147         
1148         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1149     
1150         m_indexInBlock = branchIndexInBlock;
1151         m_currentNode = branchNode;
1152         
1153         return true;
1154     }
1155     
1156     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1157     
1158     return false;
1159 }
1160
1161 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1162 {
1163     unsigned branchIndexInBlock = detectPeepHoleBranch();
1164     if (branchIndexInBlock != UINT_MAX) {
1165         Node* branchNode = m_block->at(branchIndexInBlock);
1166
1167         ASSERT(node->adjustedRefCount() == 1);
1168         
1169         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1170     
1171         m_indexInBlock = branchIndexInBlock;
1172         m_currentNode = branchNode;
1173         
1174         return true;
1175     }
1176     
1177     nonSpeculativeNonPeepholeStrictEq(node, invert);
1178     
1179     return false;
1180 }
1181
1182 static const char* dataFormatString(DataFormat format)
1183 {
1184     // These values correspond to the DataFormat enum.
1185     const char* strings[] = {
1186         "[  ]",
1187         "[ i]",
1188         "[ d]",
1189         "[ c]",
1190         "Err!",
1191         "Err!",
1192         "Err!",
1193         "Err!",
1194         "[J ]",
1195         "[Ji]",
1196         "[Jd]",
1197         "[Jc]",
1198         "Err!",
1199         "Err!",
1200         "Err!",
1201         "Err!",
1202     };
1203     return strings[format];
1204 }
1205
1206 void SpeculativeJIT::dump(const char* label)
1207 {
1208     if (label)
1209         dataLogF("<%s>\n", label);
1210
1211     dataLogF("  gprs:\n");
1212     m_gprs.dump();
1213     dataLogF("  fprs:\n");
1214     m_fprs.dump();
1215     dataLogF("  VirtualRegisters:\n");
1216     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1217         GenerationInfo& info = m_generationInfo[i];
1218         if (info.alive())
1219             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1220         else
1221             dataLogF("    % 3d:[__][__]", i);
1222         if (info.registerFormat() == DataFormatDouble)
1223             dataLogF(":fpr%d\n", info.fpr());
1224         else if (info.registerFormat() != DataFormatNone
1225 #if USE(JSVALUE32_64)
1226             && !(info.registerFormat() & DataFormatJS)
1227 #endif
1228             ) {
1229             ASSERT(info.gpr() != InvalidGPRReg);
1230             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1231         } else
1232             dataLogF("\n");
1233     }
1234     if (label)
1235         dataLogF("</%s>\n", label);
1236 }
1237
1238 GPRTemporary::GPRTemporary()
1239     : m_jit(0)
1240     , m_gpr(InvalidGPRReg)
1241 {
1242 }
1243
1244 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1245     : m_jit(jit)
1246     , m_gpr(InvalidGPRReg)
1247 {
1248     m_gpr = m_jit->allocate();
1249 }
1250
1251 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1252     : m_jit(jit)
1253     , m_gpr(InvalidGPRReg)
1254 {
1255     m_gpr = m_jit->allocate(specific);
1256 }
1257
1258 #if USE(JSVALUE32_64)
1259 GPRTemporary::GPRTemporary(
1260     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1261     : m_jit(jit)
1262     , m_gpr(InvalidGPRReg)
1263 {
1264     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1265         m_gpr = m_jit->reuse(op1.gpr(which));
1266     else
1267         m_gpr = m_jit->allocate();
1268 }
1269 #endif // USE(JSVALUE32_64)
1270
1271 JSValueRegsTemporary::JSValueRegsTemporary() { }
1272
1273 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1274 #if USE(JSVALUE64)
1275     : m_gpr(jit)
1276 #else
1277     : m_payloadGPR(jit)
1278     , m_tagGPR(jit)
1279 #endif
1280 {
1281 }
1282
1283 #if USE(JSVALUE64)
1284 template<typename T>
1285 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1286     : m_gpr(jit, Reuse, operand)
1287 {
1288 }
1289 #else
1290 template<typename T>
1291 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1292 {
1293     if (resultWord == PayloadWord) {
1294         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1295         m_tagGPR = GPRTemporary(jit);
1296     } else {
1297         m_payloadGPR = GPRTemporary(jit);
1298         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1299     }
1300 }
1301 #endif
1302
1303 #if USE(JSVALUE64)
1304 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1305 {
1306     m_gpr = GPRTemporary(jit, Reuse, operand);
1307 }
1308 #else
1309 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1310 {
1311     if (jit->canReuse(operand.node())) {
1312         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1313         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1314     } else {
1315         m_payloadGPR = GPRTemporary(jit);
1316         m_tagGPR = GPRTemporary(jit);
1317     }
1318 }
1319 #endif
1320
1321 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1322
1323 JSValueRegs JSValueRegsTemporary::regs()
1324 {
1325 #if USE(JSVALUE64)
1326     return JSValueRegs(m_gpr.gpr());
1327 #else
1328     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1329 #endif
1330 }
1331
1332 void GPRTemporary::adopt(GPRTemporary& other)
1333 {
1334     ASSERT(!m_jit);
1335     ASSERT(m_gpr == InvalidGPRReg);
1336     ASSERT(other.m_jit);
1337     ASSERT(other.m_gpr != InvalidGPRReg);
1338     m_jit = other.m_jit;
1339     m_gpr = other.m_gpr;
1340     other.m_jit = 0;
1341     other.m_gpr = InvalidGPRReg;
1342 }
1343
1344 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1345 {
1346     ASSERT(other.m_jit);
1347     ASSERT(other.m_fpr != InvalidFPRReg);
1348     m_jit = other.m_jit;
1349     m_fpr = other.m_fpr;
1350
1351     other.m_jit = nullptr;
1352 }
1353
1354 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1355     : m_jit(jit)
1356     , m_fpr(InvalidFPRReg)
1357 {
1358     m_fpr = m_jit->fprAllocate();
1359 }
1360
1361 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1362     : m_jit(jit)
1363     , m_fpr(InvalidFPRReg)
1364 {
1365     if (m_jit->canReuse(op1.node()))
1366         m_fpr = m_jit->reuse(op1.fpr());
1367     else
1368         m_fpr = m_jit->fprAllocate();
1369 }
1370
1371 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1372     : m_jit(jit)
1373     , m_fpr(InvalidFPRReg)
1374 {
1375     if (m_jit->canReuse(op1.node()))
1376         m_fpr = m_jit->reuse(op1.fpr());
1377     else if (m_jit->canReuse(op2.node()))
1378         m_fpr = m_jit->reuse(op2.fpr());
1379     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1380         m_fpr = m_jit->reuse(op1.fpr());
1381     else
1382         m_fpr = m_jit->fprAllocate();
1383 }
1384
1385 #if USE(JSVALUE32_64)
1386 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1387     : m_jit(jit)
1388     , m_fpr(InvalidFPRReg)
1389 {
1390     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1391         m_fpr = m_jit->reuse(op1.fpr());
1392     else
1393         m_fpr = m_jit->fprAllocate();
1394 }
1395 #endif
1396
1397 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1398 {
1399     BasicBlock* taken = branchNode->branchData()->taken.block;
1400     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1401
1402     if (taken == nextBlock()) {
1403         condition = MacroAssembler::invert(condition);
1404         std::swap(taken, notTaken);
1405     }
1406
1407     SpeculateDoubleOperand op1(this, node->child1());
1408     SpeculateDoubleOperand op2(this, node->child2());
1409     
1410     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1411     jump(notTaken);
1412 }
1413
1414 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1415 {
1416     BasicBlock* taken = branchNode->branchData()->taken.block;
1417     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1418
1419     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1420     
1421     if (taken == nextBlock()) {
1422         condition = MacroAssembler::NotEqual;
1423         BasicBlock* tmp = taken;
1424         taken = notTaken;
1425         notTaken = tmp;
1426     }
1427
1428     SpeculateCellOperand op1(this, node->child1());
1429     SpeculateCellOperand op2(this, node->child2());
1430     
1431     GPRReg op1GPR = op1.gpr();
1432     GPRReg op2GPR = op2.gpr();
1433     
1434     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1435         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1436             speculationCheck(
1437                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1438         }
1439         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1440             speculationCheck(
1441                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1442         }
1443     } else {
1444         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1445             speculationCheck(
1446                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1447                 m_jit.branchIfNotObject(op1GPR));
1448         }
1449         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1450             m_jit.branchTest8(
1451                 MacroAssembler::NonZero, 
1452                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1453                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1454
1455         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1456             speculationCheck(
1457                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1458                 m_jit.branchIfNotObject(op2GPR));
1459         }
1460         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1461             m_jit.branchTest8(
1462                 MacroAssembler::NonZero, 
1463                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1464                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1465     }
1466
1467     branchPtr(condition, op1GPR, op2GPR, taken);
1468     jump(notTaken);
1469 }
1470
1471 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1472 {
1473     BasicBlock* taken = branchNode->branchData()->taken.block;
1474     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1475
1476     // The branch instruction will branch to the taken block.
1477     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1478     if (taken == nextBlock()) {
1479         condition = JITCompiler::invert(condition);
1480         BasicBlock* tmp = taken;
1481         taken = notTaken;
1482         notTaken = tmp;
1483     }
1484
1485     if (node->child1()->isInt32Constant()) {
1486         int32_t imm = node->child1()->asInt32();
1487         SpeculateBooleanOperand op2(this, node->child2());
1488         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1489     } else if (node->child2()->isInt32Constant()) {
1490         SpeculateBooleanOperand op1(this, node->child1());
1491         int32_t imm = node->child2()->asInt32();
1492         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1493     } else {
1494         SpeculateBooleanOperand op1(this, node->child1());
1495         SpeculateBooleanOperand op2(this, node->child2());
1496         branch32(condition, op1.gpr(), op2.gpr(), taken);
1497     }
1498
1499     jump(notTaken);
1500 }
1501
1502 void SpeculativeJIT::compileStringSlice(Node* node)
1503 {
1504     SpeculateCellOperand string(this, node->child1());
1505     GPRTemporary startIndex(this);
1506     GPRTemporary temp(this);
1507     GPRTemporary temp2(this);
1508
1509     GPRReg stringGPR = string.gpr();
1510     GPRReg startIndexGPR = startIndex.gpr();
1511     GPRReg tempGPR = temp.gpr();
1512     GPRReg temp2GPR = temp2.gpr();
1513
1514     speculateString(node->child1(), stringGPR);
1515
1516     {
1517         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1518
1519         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1520         if (node->child3())
1521             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1522         else
1523             m_jit.move(temp2GPR, tempGPR);
1524     }
1525
1526     CCallHelpers::JumpList doneCases;
1527     CCallHelpers::JumpList slowCases;
1528
1529     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1530     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1531     doneCases.append(m_jit.jump());
1532
1533     nonEmptyCase.link(&m_jit);
1534     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1535     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1536
1537     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1538     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1539
1540     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1541
1542     // Load the character into scratchReg
1543     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1544     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1545
1546     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1547     auto cont8Bit = m_jit.jump();
1548
1549     is16Bit.link(&m_jit);
1550     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1551
1552     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1553
1554     // 8 bit string values don't need the isASCII check.
1555     cont8Bit.link(&m_jit);
1556
1557     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1558     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1559     m_jit.loadPtr(tempGPR, tempGPR);
1560
1561     addSlowPathGenerator(
1562         slowPathCall(
1563             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1564
1565     addSlowPathGenerator(
1566         slowPathCall(
1567             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1568
1569     doneCases.link(&m_jit);
1570     cellResult(tempGPR, node);
1571 }
1572
1573 void SpeculativeJIT::compileToLowerCase(Node* node)
1574 {
1575     ASSERT(node->op() == ToLowerCase);
1576     SpeculateCellOperand string(this, node->child1());
1577     GPRTemporary temp(this);
1578     GPRTemporary index(this);
1579     GPRTemporary charReg(this);
1580     GPRTemporary length(this);
1581
1582     GPRReg stringGPR = string.gpr();
1583     GPRReg tempGPR = temp.gpr();
1584     GPRReg indexGPR = index.gpr();
1585     GPRReg charGPR = charReg.gpr();
1586     GPRReg lengthGPR = length.gpr();
1587
1588     speculateString(node->child1(), stringGPR);
1589
1590     CCallHelpers::JumpList slowPath;
1591
1592     m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1593
1594     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1595     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1596
1597     slowPath.append(m_jit.branchTest32(
1598         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1599         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1600     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1601     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1602
1603     auto loopStart = m_jit.label();
1604     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1605     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1606     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1607     m_jit.sub32(TrustedImm32('A'), charGPR);
1608     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1609
1610     m_jit.add32(TrustedImm32(1), indexGPR);
1611     m_jit.jump().linkTo(loopStart, &m_jit);
1612     
1613     slowPath.link(&m_jit);
1614     silentSpillAllRegisters(lengthGPR);
1615     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1616     silentFillAllRegisters();
1617     m_jit.exceptionCheck();
1618     auto done = m_jit.jump();
1619
1620     loopDone.link(&m_jit);
1621     m_jit.move(stringGPR, lengthGPR);
1622
1623     done.link(&m_jit);
1624     cellResult(lengthGPR, node);
1625 }
1626
1627 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1628 {
1629     BasicBlock* taken = branchNode->branchData()->taken.block;
1630     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1631
1632     // The branch instruction will branch to the taken block.
1633     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1634     if (taken == nextBlock()) {
1635         condition = JITCompiler::invert(condition);
1636         BasicBlock* tmp = taken;
1637         taken = notTaken;
1638         notTaken = tmp;
1639     }
1640
1641     if (node->child1()->isInt32Constant()) {
1642         int32_t imm = node->child1()->asInt32();
1643         SpeculateInt32Operand op2(this, node->child2());
1644         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1645     } else if (node->child2()->isInt32Constant()) {
1646         SpeculateInt32Operand op1(this, node->child1());
1647         int32_t imm = node->child2()->asInt32();
1648         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1649     } else {
1650         SpeculateInt32Operand op1(this, node->child1());
1651         SpeculateInt32Operand op2(this, node->child2());
1652         branch32(condition, op1.gpr(), op2.gpr(), taken);
1653     }
1654
1655     jump(notTaken);
1656 }
1657
1658 // Returns true if the compare is fused with a subsequent branch.
1659 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1660 {
1661     // Fused compare & branch.
1662     unsigned branchIndexInBlock = detectPeepHoleBranch();
1663     if (branchIndexInBlock != UINT_MAX) {
1664         Node* branchNode = m_block->at(branchIndexInBlock);
1665
1666         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1667         // so can be no intervening nodes to also reference the compare. 
1668         ASSERT(node->adjustedRefCount() == 1);
1669
1670         if (node->isBinaryUseKind(Int32Use))
1671             compilePeepHoleInt32Branch(node, branchNode, condition);
1672 #if USE(JSVALUE64)
1673         else if (node->isBinaryUseKind(Int52RepUse))
1674             compilePeepHoleInt52Branch(node, branchNode, condition);
1675 #endif // USE(JSVALUE64)
1676         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1677             // Use non-peephole comparison, for now.
1678             return false;
1679         } else if (node->isBinaryUseKind(DoubleRepUse))
1680             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1681         else if (node->op() == CompareEq) {
1682             if (node->isBinaryUseKind(BooleanUse))
1683                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1684             else if (node->isBinaryUseKind(SymbolUse))
1685                 compilePeepHoleSymbolEquality(node, branchNode);
1686             else if (node->isBinaryUseKind(ObjectUse))
1687                 compilePeepHoleObjectEquality(node, branchNode);
1688             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1689                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1690             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1691                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1692             else if (!needsTypeCheck(node->child1(), SpecOther))
1693                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1694             else if (!needsTypeCheck(node->child2(), SpecOther))
1695                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1696             else {
1697                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1698                 return true;
1699             }
1700         } else {
1701             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1702             return true;
1703         }
1704
1705         use(node->child1());
1706         use(node->child2());
1707         m_indexInBlock = branchIndexInBlock;
1708         m_currentNode = branchNode;
1709         return true;
1710     }
1711     return false;
1712 }
1713
1714 void SpeculativeJIT::noticeOSRBirth(Node* node)
1715 {
1716     if (!node->hasVirtualRegister())
1717         return;
1718     
1719     VirtualRegister virtualRegister = node->virtualRegister();
1720     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1721     
1722     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1723 }
1724
1725 void SpeculativeJIT::compileMovHint(Node* node)
1726 {
1727     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1728     
1729     Node* child = node->child1().node();
1730     noticeOSRBirth(child);
1731     
1732     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1733 }
1734
1735 void SpeculativeJIT::bail(AbortReason reason)
1736 {
1737     if (verboseCompilationEnabled())
1738         dataLog("Bailing compilation.\n");
1739     m_compileOkay = true;
1740     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1741     clearGenerationInfo();
1742 }
1743
1744 void SpeculativeJIT::compileCurrentBlock()
1745 {
1746     ASSERT(m_compileOkay);
1747     
1748     if (!m_block)
1749         return;
1750     
1751     ASSERT(m_block->isReachable);
1752     
1753     m_jit.blockHeads()[m_block->index] = m_jit.label();
1754
1755     if (!m_block->intersectionOfCFAHasVisited) {
1756         // Don't generate code for basic blocks that are unreachable according to CFA.
1757         // But to be sure that nobody has generated a jump to this block, drop in a
1758         // breakpoint here.
1759         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1760         return;
1761     }
1762
1763     if (m_block->isCatchEntrypoint) {
1764         m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1765         if (Options::zeroStackFrame())
1766             m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1767         m_jit.emitSaveCalleeSaves();
1768         m_jit.emitMaterializeTagCheckRegisters();
1769         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1770     }
1771
1772     m_stream->appendAndLog(VariableEvent::reset());
1773     
1774     m_jit.jitAssertHasValidCallFrame();
1775     m_jit.jitAssertTagsInPlace();
1776     m_jit.jitAssertArgumentCountSane();
1777
1778     m_state.reset();
1779     m_state.beginBasicBlock(m_block);
1780     
1781     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1782         int operand = m_block->variablesAtHead.operandForIndex(i);
1783         Node* node = m_block->variablesAtHead[i];
1784         if (!node)
1785             continue; // No need to record dead SetLocal's.
1786         
1787         VariableAccessData* variable = node->variableAccessData();
1788         DataFormat format;
1789         if (!node->refCount())
1790             continue; // No need to record dead SetLocal's.
1791         format = dataFormatFor(variable->flushFormat());
1792         m_stream->appendAndLog(
1793             VariableEvent::setLocal(
1794                 VirtualRegister(operand),
1795                 variable->machineLocal(),
1796                 format));
1797     }
1798
1799     m_origin = NodeOrigin();
1800     
1801     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1802         m_currentNode = m_block->at(m_indexInBlock);
1803         
1804         // We may have hit a contradiction that the CFA was aware of but that the JIT
1805         // didn't cause directly.
1806         if (!m_state.isValid()) {
1807             bail(DFGBailedAtTopOfBlock);
1808             return;
1809         }
1810
1811         m_interpreter.startExecuting();
1812         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1813         m_jit.setForNode(m_currentNode);
1814         m_origin = m_currentNode->origin;
1815         if (validationEnabled())
1816             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1817         m_lastGeneratedNode = m_currentNode->op();
1818         
1819         ASSERT(m_currentNode->shouldGenerate());
1820         
1821         if (verboseCompilationEnabled()) {
1822             dataLogF(
1823                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1824                 (int)m_currentNode->index(),
1825                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1826             dataLog("\n");
1827         }
1828
1829         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1830             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1831
1832         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1833
1834         compile(m_currentNode);
1835         
1836         if (belongsInMinifiedGraph(m_currentNode->op()))
1837             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1838         
1839 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1840         m_jit.clearRegisterAllocationOffsets();
1841 #endif
1842         
1843         if (!m_compileOkay) {
1844             bail(DFGBailedAtEndOfNode);
1845             return;
1846         }
1847         
1848         // Make sure that the abstract state is rematerialized for the next node.
1849         m_interpreter.executeEffects(m_indexInBlock);
1850     }
1851     
1852     // Perform the most basic verification that children have been used correctly.
1853     if (!ASSERT_DISABLED) {
1854         for (auto& info : m_generationInfo)
1855             RELEASE_ASSERT(!info.alive());
1856     }
1857 }
1858
1859 // If we are making type predictions about our arguments then
1860 // we need to check that they are correct on function entry.
1861 void SpeculativeJIT::checkArgumentTypes()
1862 {
1863     ASSERT(!m_currentNode);
1864     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1865
1866     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1867     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1868         Node* node = arguments[i];
1869         if (!node) {
1870             // The argument is dead. We don't do any checks for such arguments.
1871             continue;
1872         }
1873         
1874         ASSERT(node->op() == SetArgument);
1875         ASSERT(node->shouldGenerate());
1876
1877         VariableAccessData* variableAccessData = node->variableAccessData();
1878         FlushFormat format = variableAccessData->flushFormat();
1879         
1880         if (format == FlushedJSValue)
1881             continue;
1882         
1883         VirtualRegister virtualRegister = variableAccessData->local();
1884
1885         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1886         
1887 #if USE(JSVALUE64)
1888         switch (format) {
1889         case FlushedInt32: {
1890             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1891             break;
1892         }
1893         case FlushedBoolean: {
1894             GPRTemporary temp(this);
1895             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1896             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1897             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1898             break;
1899         }
1900         case FlushedCell: {
1901             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1902             break;
1903         }
1904         default:
1905             RELEASE_ASSERT_NOT_REACHED();
1906             break;
1907         }
1908 #else
1909         switch (format) {
1910         case FlushedInt32: {
1911             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1912             break;
1913         }
1914         case FlushedBoolean: {
1915             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1916             break;
1917         }
1918         case FlushedCell: {
1919             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1920             break;
1921         }
1922         default:
1923             RELEASE_ASSERT_NOT_REACHED();
1924             break;
1925         }
1926 #endif
1927     }
1928
1929     m_origin = NodeOrigin();
1930 }
1931
1932 bool SpeculativeJIT::compile()
1933 {
1934     checkArgumentTypes();
1935     
1936     ASSERT(!m_currentNode);
1937     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1938         m_jit.setForBlockIndex(blockIndex);
1939         m_block = m_jit.graph().block(blockIndex);
1940         compileCurrentBlock();
1941     }
1942     linkBranches();
1943     return true;
1944 }
1945
1946 void SpeculativeJIT::createOSREntries()
1947 {
1948     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1949         BasicBlock* block = m_jit.graph().block(blockIndex);
1950         if (!block)
1951             continue;
1952         if (block->isOSRTarget || block->isCatchEntrypoint) {
1953             // Currently we don't have OSR entry trampolines. We could add them
1954             // here if need be.
1955             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1956         }
1957     }
1958 }
1959
1960 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1961 {
1962     unsigned osrEntryIndex = 0;
1963     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1964         BasicBlock* block = m_jit.graph().block(blockIndex);
1965         if (!block)
1966             continue;
1967         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1968             continue;
1969         if (block->isCatchEntrypoint) {
1970             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1971             Vector<FlushFormat> argumentFormats;
1972             argumentFormats.reserveInitialCapacity(argumentsVector.size());
1973             for (Node* setArgument : argumentsVector) {
1974                 if (setArgument) {
1975                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
1976                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
1977                     argumentFormats.uncheckedAppend(flushFormat);
1978                 } else
1979                     argumentFormats.uncheckedAppend(DeadFlush);
1980             }
1981             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
1982         } else {
1983             ASSERT(block->isOSRTarget);
1984             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
1985         }
1986     }
1987
1988     m_jit.jitCode()->finalizeOSREntrypoints();
1989     m_jit.jitCode()->common.finalizeCatchEntrypoints();
1990
1991     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
1992     
1993     if (verboseCompilationEnabled()) {
1994         DumpContext dumpContext;
1995         dataLog("OSR Entries:\n");
1996         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
1997             dataLog("    ", inContext(entryData, &dumpContext), "\n");
1998         if (!dumpContext.isEmpty())
1999             dumpContext.dump(WTF::dataFile());
2000     }
2001 }
2002     
2003 void SpeculativeJIT::compileCheckTraps(Node*)
2004 {
2005     ASSERT(Options::usePollingTraps());
2006     GPRTemporary unused(this);
2007     GPRReg unusedGPR = unused.gpr();
2008
2009     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2010         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2011
2012     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2013 }
2014
2015 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2016 {
2017     Edge child3 = m_jit.graph().varArgChild(node, 2);
2018     Edge child4 = m_jit.graph().varArgChild(node, 3);
2019
2020     ArrayMode arrayMode = node->arrayMode();
2021     
2022     GPRReg baseReg = base.gpr();
2023     GPRReg propertyReg = property.gpr();
2024     
2025     SpeculateDoubleOperand value(this, child3);
2026
2027     FPRReg valueReg = value.fpr();
2028     
2029     DFG_TYPE_CHECK(
2030         JSValueRegs(), child3, SpecFullRealNumber,
2031         m_jit.branchDouble(
2032             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2033     
2034     if (!m_compileOkay)
2035         return;
2036     
2037     StorageOperand storage(this, child4);
2038     GPRReg storageReg = storage.gpr();
2039
2040     if (node->op() == PutByValAlias) {
2041         // Store the value to the array.
2042         GPRReg propertyReg = property.gpr();
2043         FPRReg valueReg = value.fpr();
2044         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2045         
2046         noResult(m_currentNode);
2047         return;
2048     }
2049     
2050     GPRTemporary temporary;
2051     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2052
2053     MacroAssembler::Jump slowCase;
2054     
2055     if (arrayMode.isInBounds()) {
2056         speculationCheck(
2057             OutOfBounds, JSValueRegs(), 0,
2058             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2059     } else {
2060         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2061         
2062         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2063         
2064         if (!arrayMode.isOutOfBounds())
2065             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2066         
2067         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2068         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2069
2070         inBounds.link(&m_jit);
2071     }
2072     
2073     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2074
2075     base.use();
2076     property.use();
2077     value.use();
2078     storage.use();
2079     
2080     if (arrayMode.isOutOfBounds()) {
2081         addSlowPathGenerator(
2082             slowPathCall(
2083                 slowCase, this,
2084                 m_jit.codeBlock()->isStrictMode()
2085                     ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2086                     : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2087                 NoResult, baseReg, propertyReg, valueReg));
2088     }
2089
2090     noResult(m_currentNode, UseChildrenCalledExplicitly);
2091 }
2092
2093 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2094 {
2095     SpeculateCellOperand string(this, node->child1());
2096     SpeculateStrictInt32Operand index(this, node->child2());
2097     StorageOperand storage(this, node->child3());
2098
2099     GPRReg stringReg = string.gpr();
2100     GPRReg indexReg = index.gpr();
2101     GPRReg storageReg = storage.gpr();
2102     
2103     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2104
2105     // unsigned comparison so we can filter out negative indices and indices that are too large
2106     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2107
2108     GPRTemporary scratch(this);
2109     GPRReg scratchReg = scratch.gpr();
2110
2111     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2112     m_jit.and32(MacroAssembler::Address(scratchReg, StringImpl::maskOffset()), indexReg);
2113
2114     // Load the character into scratchReg
2115     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2116
2117     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2118     JITCompiler::Jump cont8Bit = m_jit.jump();
2119
2120     is16Bit.link(&m_jit);
2121
2122     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2123
2124     cont8Bit.link(&m_jit);
2125
2126     int32Result(scratchReg, m_currentNode);
2127 }
2128
2129 void SpeculativeJIT::compileGetByValOnString(Node* node)
2130 {
2131     SpeculateCellOperand base(this, m_graph.child(node, 0));
2132     SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2133     StorageOperand storage(this, m_graph.child(node, 2));
2134     GPRReg baseReg = base.gpr();
2135     GPRReg propertyReg = property.gpr();
2136     GPRReg storageReg = storage.gpr();
2137
2138     GPRTemporary scratch(this);
2139     GPRReg scratchReg = scratch.gpr();
2140 #if USE(JSVALUE32_64)
2141     GPRTemporary resultTag;
2142     GPRReg resultTagReg = InvalidGPRReg;
2143     if (node->arrayMode().isOutOfBounds()) {
2144         GPRTemporary realResultTag(this);
2145         resultTag.adopt(realResultTag);
2146         resultTagReg = resultTag.gpr();
2147     }
2148 #endif
2149
2150     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2151
2152     // unsigned comparison so we can filter out negative indices and indices that are too large
2153     JITCompiler::Jump outOfBounds = m_jit.branch32(
2154         MacroAssembler::AboveOrEqual, propertyReg,
2155         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2156     if (node->arrayMode().isInBounds())
2157         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2158
2159     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2160     m_jit.and32(MacroAssembler::Address(scratchReg, StringImpl::maskOffset()), propertyReg);
2161
2162     // Load the character into scratchReg
2163     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2164
2165     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2166     JITCompiler::Jump cont8Bit = m_jit.jump();
2167
2168     is16Bit.link(&m_jit);
2169
2170     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2171
2172     JITCompiler::Jump bigCharacter =
2173         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2174
2175     // 8 bit string values don't need the isASCII check.
2176     cont8Bit.link(&m_jit);
2177
2178     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2179     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2180     m_jit.loadPtr(scratchReg, scratchReg);
2181
2182     addSlowPathGenerator(
2183         slowPathCall(
2184             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2185
2186     if (node->arrayMode().isOutOfBounds()) {
2187 #if USE(JSVALUE32_64)
2188         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2189 #endif
2190
2191         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2192         bool prototypeChainIsSane = false;
2193         if (globalObject->stringPrototypeChainIsSane()) {
2194             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2195             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2196             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2197             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2198             // indexed properties either.
2199             // https://bugs.webkit.org/show_bug.cgi?id=144668
2200             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure());
2201             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure());
2202             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2203         }
2204         if (prototypeChainIsSane) {
2205 #if USE(JSVALUE64)
2206             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2207                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2208 #else
2209             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2210                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2211                 baseReg, propertyReg));
2212 #endif
2213         } else {
2214 #if USE(JSVALUE64)
2215             addSlowPathGenerator(
2216                 slowPathCall(
2217                     outOfBounds, this, operationGetByValStringInt,
2218                     scratchReg, baseReg, propertyReg));
2219 #else
2220             addSlowPathGenerator(
2221                 slowPathCall(
2222                     outOfBounds, this, operationGetByValStringInt,
2223                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2224 #endif
2225         }
2226         
2227 #if USE(JSVALUE64)
2228         jsValueResult(scratchReg, m_currentNode);
2229 #else
2230         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2231 #endif
2232     } else
2233         cellResult(scratchReg, m_currentNode);
2234 }
2235
2236 void SpeculativeJIT::compileFromCharCode(Node* node)
2237 {
2238     Edge& child = node->child1();
2239     if (child.useKind() == UntypedUse) {
2240         JSValueOperand opr(this, child);
2241         JSValueRegs oprRegs = opr.jsValueRegs();
2242
2243         flushRegisters();
2244         JSValueRegsFlushedCallResult result(this);
2245         JSValueRegs resultRegs = result.regs();
2246         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2247         m_jit.exceptionCheck();
2248         
2249         jsValueResult(resultRegs, node);
2250         return;
2251     }
2252
2253     SpeculateStrictInt32Operand property(this, child);
2254     GPRReg propertyReg = property.gpr();
2255     GPRTemporary smallStrings(this);
2256     GPRTemporary scratch(this);
2257     GPRReg scratchReg = scratch.gpr();
2258     GPRReg smallStringsReg = smallStrings.gpr();
2259
2260     JITCompiler::JumpList slowCases;
2261     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2262     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2263     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2264
2265     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2266     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2267     cellResult(scratchReg, m_currentNode);
2268 }
2269
2270 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2271 {
2272     VirtualRegister virtualRegister = node->virtualRegister();
2273     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2274
2275     switch (info.registerFormat()) {
2276     case DataFormatStorage:
2277         RELEASE_ASSERT_NOT_REACHED();
2278
2279     case DataFormatBoolean:
2280     case DataFormatCell:
2281         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2282         return GeneratedOperandTypeUnknown;
2283
2284     case DataFormatNone:
2285     case DataFormatJSCell:
2286     case DataFormatJS:
2287     case DataFormatJSBoolean:
2288     case DataFormatJSDouble:
2289         return GeneratedOperandJSValue;
2290
2291     case DataFormatJSInt32:
2292     case DataFormatInt32:
2293         return GeneratedOperandInteger;
2294
2295     default:
2296         RELEASE_ASSERT_NOT_REACHED();
2297         return GeneratedOperandTypeUnknown;
2298     }
2299 }
2300
2301 void SpeculativeJIT::compileValueToInt32(Node* node)
2302 {
2303     switch (node->child1().useKind()) {
2304 #if USE(JSVALUE64)
2305     case Int52RepUse: {
2306         SpeculateStrictInt52Operand op1(this, node->child1());
2307         GPRTemporary result(this, Reuse, op1);
2308         GPRReg op1GPR = op1.gpr();
2309         GPRReg resultGPR = result.gpr();
2310         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2311         int32Result(resultGPR, node, DataFormatInt32);
2312         return;
2313     }
2314 #endif // USE(JSVALUE64)
2315         
2316     case DoubleRepUse: {
2317         GPRTemporary result(this);
2318         SpeculateDoubleOperand op1(this, node->child1());
2319         FPRReg fpr = op1.fpr();
2320         GPRReg gpr = result.gpr();
2321         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2322         
2323         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2324             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2325         
2326         int32Result(gpr, node);
2327         return;
2328     }
2329     
2330     case NumberUse:
2331     case NotCellUse: {
2332         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2333         case GeneratedOperandInteger: {
2334             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2335             GPRTemporary result(this, Reuse, op1);
2336             m_jit.move(op1.gpr(), result.gpr());
2337             int32Result(result.gpr(), node, op1.format());
2338             return;
2339         }
2340         case GeneratedOperandJSValue: {
2341             GPRTemporary result(this);
2342 #if USE(JSVALUE64)
2343             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2344
2345             GPRReg gpr = op1.gpr();
2346             GPRReg resultGpr = result.gpr();
2347             FPRTemporary tempFpr(this);
2348             FPRReg fpr = tempFpr.fpr();
2349
2350             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2351             JITCompiler::JumpList converted;
2352
2353             if (node->child1().useKind() == NumberUse) {
2354                 DFG_TYPE_CHECK(
2355                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2356                     m_jit.branchTest64(
2357                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2358             } else {
2359                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2360                 
2361                 DFG_TYPE_CHECK(
2362                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2363                 
2364                 // It's not a cell: so true turns into 1 and all else turns into 0.
2365                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2366                 converted.append(m_jit.jump());
2367                 
2368                 isNumber.link(&m_jit);
2369             }
2370
2371             // First, if we get here we have a double encoded as a JSValue
2372             unboxDouble(gpr, resultGpr, fpr);
2373
2374             silentSpillAllRegisters(resultGpr);
2375             callOperation(operationToInt32, resultGpr, fpr);
2376             silentFillAllRegisters();
2377
2378             converted.append(m_jit.jump());
2379
2380             isInteger.link(&m_jit);
2381             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2382
2383             converted.link(&m_jit);
2384 #else
2385             Node* childNode = node->child1().node();
2386             VirtualRegister virtualRegister = childNode->virtualRegister();
2387             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2388
2389             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2390
2391             GPRReg payloadGPR = op1.payloadGPR();
2392             GPRReg resultGpr = result.gpr();
2393         
2394             JITCompiler::JumpList converted;
2395
2396             if (info.registerFormat() == DataFormatJSInt32)
2397                 m_jit.move(payloadGPR, resultGpr);
2398             else {
2399                 GPRReg tagGPR = op1.tagGPR();
2400                 FPRTemporary tempFpr(this);
2401                 FPRReg fpr = tempFpr.fpr();
2402                 FPRTemporary scratch(this);
2403
2404                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2405
2406                 if (node->child1().useKind() == NumberUse) {
2407                     DFG_TYPE_CHECK(
2408                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2409                         m_jit.branch32(
2410                             MacroAssembler::AboveOrEqual, tagGPR,
2411                             TrustedImm32(JSValue::LowestTag)));
2412                 } else {
2413                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2414                     
2415                     DFG_TYPE_CHECK(
2416                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2417                         m_jit.branchIfCell(op1.jsValueRegs()));
2418                     
2419                     // It's not a cell: so true turns into 1 and all else turns into 0.
2420                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2421                     m_jit.move(TrustedImm32(0), resultGpr);
2422                     converted.append(m_jit.jump());
2423                     
2424                     isBoolean.link(&m_jit);
2425                     m_jit.move(payloadGPR, resultGpr);
2426                     converted.append(m_jit.jump());
2427                     
2428                     isNumber.link(&m_jit);
2429                 }
2430
2431                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2432
2433                 silentSpillAllRegisters(resultGpr);
2434                 callOperation(operationToInt32, resultGpr, fpr);
2435                 silentFillAllRegisters();
2436
2437                 converted.append(m_jit.jump());
2438
2439                 isInteger.link(&m_jit);
2440                 m_jit.move(payloadGPR, resultGpr);
2441
2442                 converted.link(&m_jit);
2443             }
2444 #endif
2445             int32Result(resultGpr, node);
2446             return;
2447         }
2448         case GeneratedOperandTypeUnknown:
2449             RELEASE_ASSERT(!m_compileOkay);
2450             return;
2451         }
2452         RELEASE_ASSERT_NOT_REACHED();
2453         return;
2454     }
2455     
2456     default:
2457         ASSERT(!m_compileOkay);
2458         return;
2459     }
2460 }
2461
2462 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2463 {
2464     if (doesOverflow(node->arithMode())) {
2465         if (enableInt52()) {
2466             SpeculateInt32Operand op1(this, node->child1());
2467             GPRTemporary result(this, Reuse, op1);
2468             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2469             strictInt52Result(result.gpr(), node);
2470             return;
2471         }
2472         SpeculateInt32Operand op1(this, node->child1());
2473         FPRTemporary result(this);
2474             
2475         GPRReg inputGPR = op1.gpr();
2476         FPRReg outputFPR = result.fpr();
2477             
2478         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2479             
2480         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2481         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2482         positive.link(&m_jit);
2483             
2484         doubleResult(outputFPR, node);
2485         return;
2486     }
2487     
2488     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2489
2490     SpeculateInt32Operand op1(this, node->child1());
2491     GPRTemporary result(this);
2492
2493     m_jit.move(op1.gpr(), result.gpr());
2494
2495     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2496
2497     int32Result(result.gpr(), node, op1.format());
2498 }
2499
2500 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2501 {
2502     SpeculateDoubleOperand op1(this, node->child1());
2503     FPRTemporary scratch(this);
2504     GPRTemporary result(this);
2505     
2506     FPRReg valueFPR = op1.fpr();
2507     FPRReg scratchFPR = scratch.fpr();
2508     GPRReg resultGPR = result.gpr();
2509
2510     JITCompiler::JumpList failureCases;
2511     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2512     m_jit.branchConvertDoubleToInt32(
2513         valueFPR, resultGPR, failureCases, scratchFPR,
2514         shouldCheckNegativeZero(node->arithMode()));
2515     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2516
2517     int32Result(resultGPR, node);
2518 }
2519
2520 void SpeculativeJIT::compileDoubleRep(Node* node)
2521 {
2522     switch (node->child1().useKind()) {
2523     case RealNumberUse: {
2524         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2525         FPRTemporary result(this);
2526         
2527         JSValueRegs op1Regs = op1.jsValueRegs();
2528         FPRReg resultFPR = result.fpr();
2529         
2530 #if USE(JSVALUE64)
2531         GPRTemporary temp(this);
2532         GPRReg tempGPR = temp.gpr();
2533         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2534 #else
2535         FPRTemporary temp(this);
2536         FPRReg tempFPR = temp.fpr();
2537         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2538 #endif
2539         
2540         JITCompiler::Jump done = m_jit.branchDouble(
2541             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2542         
2543         DFG_TYPE_CHECK(
2544             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2545         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2546         
2547         done.link(&m_jit);
2548         
2549         doubleResult(resultFPR, node);
2550         return;
2551     }
2552     
2553     case NotCellUse:
2554     case NumberUse: {
2555         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2556
2557         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2558         if (isInt32Speculation(possibleTypes)) {
2559             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2560             FPRTemporary result(this);
2561             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2562             doubleResult(result.fpr(), node);
2563             return;
2564         }
2565
2566         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2567         FPRTemporary result(this);
2568
2569 #if USE(JSVALUE64)
2570         GPRTemporary temp(this);
2571
2572         GPRReg op1GPR = op1.gpr();
2573         GPRReg tempGPR = temp.gpr();
2574         FPRReg resultFPR = result.fpr();
2575         JITCompiler::JumpList done;
2576
2577         JITCompiler::Jump isInteger = m_jit.branch64(
2578             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2579
2580         if (node->child1().useKind() == NotCellUse) {
2581             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2582             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2583
2584             static const double zero = 0;
2585             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2586
2587             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2588             done.append(isNull);
2589
2590             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2591                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2592
2593             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2594             static const double one = 1;
2595             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2596             done.append(m_jit.jump());
2597             done.append(isFalse);
2598
2599             isUndefined.link(&m_jit);
2600             static const double NaN = PNaN;
2601             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2602             done.append(m_jit.jump());
2603
2604             isNumber.link(&m_jit);
2605         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2606             typeCheck(
2607                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2608                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2609         }
2610
2611         unboxDouble(op1GPR, tempGPR, resultFPR);
2612         done.append(m_jit.jump());
2613     
2614         isInteger.link(&m_jit);
2615         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2616         done.link(&m_jit);
2617 #else // USE(JSVALUE64) -> this is the 32_64 case
2618         FPRTemporary temp(this);
2619     
2620         GPRReg op1TagGPR = op1.tagGPR();
2621         GPRReg op1PayloadGPR = op1.payloadGPR();
2622         FPRReg tempFPR = temp.fpr();
2623         FPRReg resultFPR = result.fpr();
2624         JITCompiler::JumpList done;
2625     
2626         JITCompiler::Jump isInteger = m_jit.branch32(
2627             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2628
2629         if (node->child1().useKind() == NotCellUse) {
2630             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2631             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2632
2633             static const double zero = 0;
2634             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2635
2636             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2637             done.append(isNull);
2638
2639             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2640
2641             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2642             static const double one = 1;
2643             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2644             done.append(m_jit.jump());
2645             done.append(isFalse);
2646
2647             isUndefined.link(&m_jit);
2648             static const double NaN = PNaN;
2649             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2650             done.append(m_jit.jump());
2651
2652             isNumber.link(&m_jit);
2653         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2654             typeCheck(
2655                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2656                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2657         }
2658
2659         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2660         done.append(m_jit.jump());
2661     
2662         isInteger.link(&m_jit);
2663         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2664         done.link(&m_jit);
2665 #endif // USE(JSVALUE64)
2666     
2667         doubleResult(resultFPR, node);
2668         return;
2669     }
2670         
2671 #if USE(JSVALUE64)
2672     case Int52RepUse: {
2673         SpeculateStrictInt52Operand value(this, node->child1());
2674         FPRTemporary result(this);
2675         
2676         GPRReg valueGPR = value.gpr();
2677         FPRReg resultFPR = result.fpr();
2678
2679         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2680         
2681         doubleResult(resultFPR, node);
2682         return;
2683     }
2684 #endif // USE(JSVALUE64)
2685         
2686     default:
2687         RELEASE_ASSERT_NOT_REACHED();
2688         return;
2689     }
2690 }
2691
2692 void SpeculativeJIT::compileValueRep(Node* node)
2693 {
2694     switch (node->child1().useKind()) {
2695     case DoubleRepUse: {
2696         SpeculateDoubleOperand value(this, node->child1());
2697         JSValueRegsTemporary result(this);
2698         
2699         FPRReg valueFPR = value.fpr();
2700         JSValueRegs resultRegs = result.regs();
2701         
2702         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2703         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2704         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2705         // local was purified.
2706         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2707             m_jit.purifyNaN(valueFPR);
2708
2709         boxDouble(valueFPR, resultRegs);
2710         
2711         jsValueResult(resultRegs, node);
2712         return;
2713     }
2714         
2715 #if USE(JSVALUE64)
2716     case Int52RepUse: {
2717         SpeculateStrictInt52Operand value(this, node->child1());
2718         GPRTemporary result(this);
2719         
2720         GPRReg valueGPR = value.gpr();
2721         GPRReg resultGPR = result.gpr();
2722         
2723         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2724         
2725         jsValueResult(resultGPR, node);
2726         return;
2727     }
2728 #endif // USE(JSVALUE64)
2729         
2730     default:
2731         RELEASE_ASSERT_NOT_REACHED();
2732         return;
2733     }
2734 }
2735
2736 static double clampDoubleToByte(double d)
2737 {
2738     d += 0.5;
2739     if (!(d > 0))
2740         d = 0;
2741     else if (d > 255)
2742         d = 255;
2743     return d;
2744 }
2745
2746 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2747 {
2748     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2749     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2750     jit.xorPtr(result, result);
2751     MacroAssembler::Jump clamped = jit.jump();
2752     tooBig.link(&jit);
2753     jit.move(JITCompiler::TrustedImm32(255), result);
2754     clamped.link(&jit);
2755     inBounds.link(&jit);
2756 }
2757
2758 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2759 {
2760     // Unordered compare so we pick up NaN
2761     static const double zero = 0;
2762     static const double byteMax = 255;
2763     static const double half = 0.5;
2764     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2765     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2766     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2767     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2768     
2769     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2770     // FIXME: This should probably just use a floating point round!
2771     // https://bugs.webkit.org/show_bug.cgi?id=72054
2772     jit.addDouble(source, scratch);
2773     jit.truncateDoubleToInt32(scratch, result);   
2774     MacroAssembler::Jump truncatedInt = jit.jump();
2775     
2776     tooSmall.link(&jit);
2777     jit.xorPtr(result, result);
2778     MacroAssembler::Jump zeroed = jit.jump();
2779     
2780     tooBig.link(&jit);
2781     jit.move(JITCompiler::TrustedImm32(255), result);
2782     
2783     truncatedInt.link(&jit);
2784     zeroed.link(&jit);
2785
2786 }
2787
2788 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2789 {
2790     if (node->op() == PutByValAlias)
2791         return JITCompiler::Jump();
2792     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2793         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2794     if (view) {
2795         uint32_t length = view->length();
2796         Node* indexNode = m_jit.graph().child(node, 1).node();
2797         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2798             return JITCompiler::Jump();
2799         return m_jit.branch32(
2800             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2801     }
2802     return m_jit.branch32(
2803         MacroAssembler::AboveOrEqual, indexGPR,
2804         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2805 }
2806
2807 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2808 {
2809     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2810     if (!jump.isSet())
2811         return;
2812     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2813 }
2814
2815 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2816 {
2817     JITCompiler::Jump done;
2818     if (outOfBounds.isSet()) {
2819         done = m_jit.jump();
2820         if (node->arrayMode().isInBounds())
2821             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2822         else {
2823             outOfBounds.link(&m_jit);
2824
2825             JITCompiler::Jump notWasteful = m_jit.branch32(
2826                 MacroAssembler::NotEqual,
2827                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2828                 TrustedImm32(WastefulTypedArray));
2829
2830             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2831                 MacroAssembler::Zero,
2832                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfPoisonedVector()));
2833             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2834             notWasteful.link(&m_jit);
2835         }
2836     }
2837     return done;
2838 }
2839
2840 void SpeculativeJIT::loadFromIntTypedArray(GPRReg baseReg, GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2841 {
2842     if (m_indexMaskingMode == IndexMaskingEnabled)
2843         m_jit.and32(MacroAssembler::Address(baseReg, JSObject::butterflyIndexingMaskOffset()), propertyReg);
2844     switch (elementSize(type)) {
2845     case 1:
2846         if (isSigned(type))
2847             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2848         else
2849             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2850         break;
2851     case 2:
2852         if (isSigned(type))
2853             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2854         else
2855             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2856         break;
2857     case 4:
2858         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2859         break;
2860     default:
2861         CRASH();
2862     }
2863 }
2864
2865 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2866 {
2867     if (elementSize(type) < 4 || isSigned(type)) {
2868         int32Result(resultReg, node);
2869         return;
2870     }
2871     
2872     ASSERT(elementSize(type) == 4 && !isSigned(type));
2873     if (node->shouldSpeculateInt32() && canSpeculate) {
2874         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2875         int32Result(resultReg, node);
2876         return;
2877     }
2878     
2879 #if USE(JSVALUE64)
2880     if (node->shouldSpeculateAnyInt()) {
2881         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2882         strictInt52Result(resultReg, node);
2883         return;
2884     }
2885 #endif
2886     
2887     FPRTemporary fresult(this);
2888     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2889     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2890     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2891     positive.link(&m_jit);
2892     doubleResult(fresult.fpr(), node);
2893 }
2894
2895 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2896 {
2897     ASSERT(isInt(type));
2898     
2899     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2900     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2901     StorageOperand storage(this, m_graph.varArgChild(node, 2));
2902
2903     GPRReg baseReg = base.gpr();
2904     GPRReg propertyReg = property.gpr();
2905     GPRReg storageReg = storage.gpr();
2906
2907     GPRTemporary result(this);
2908     GPRReg resultReg = result.gpr();
2909
2910     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2911
2912     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2913     loadFromIntTypedArray(baseReg, storageReg, propertyReg, resultReg, type);
2914     bool canSpeculate = true;
2915     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2916 }
2917
2918 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2919     GPRTemporary& value,
2920     GPRReg property,
2921 #if USE(JSVALUE32_64)
2922     GPRTemporary& propertyTag,
2923     GPRTemporary& valueTag,
2924 #endif
2925     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2926 {
2927     bool isAppropriateConstant = false;
2928     if (valueUse->isConstant()) {
2929         JSValue jsValue = valueUse->asJSValue();
2930         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2931         SpeculatedType actualType = speculationFromValue(jsValue);
2932         isAppropriateConstant = (expectedType | actualType) == expectedType;
2933     }
2934     
2935     if (isAppropriateConstant) {
2936         JSValue jsValue = valueUse->asJSValue();
2937         if (!jsValue.isNumber()) {
2938             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2939             return false;
2940         }
2941         double d = jsValue.asNumber();
2942         if (isClamped)
2943             d = clampDoubleToByte(d);
2944         GPRTemporary scratch(this);
2945         GPRReg scratchReg = scratch.gpr();
2946         m_jit.move(Imm32(toInt32(d)), scratchReg);
2947         value.adopt(scratch);
2948     } else {
2949         switch (valueUse.useKind()) {
2950         case Int32Use: {
2951             SpeculateInt32Operand valueOp(this, valueUse);
2952             GPRTemporary scratch(this);
2953             GPRReg scratchReg = scratch.gpr();
2954             m_jit.move(valueOp.gpr(), scratchReg);
2955             if (isClamped)
2956                 compileClampIntegerToByte(m_jit, scratchReg);
2957             value.adopt(scratch);
2958             break;
2959         }
2960             
2961 #if USE(JSVALUE64)
2962         case Int52RepUse: {
2963             SpeculateStrictInt52Operand valueOp(this, valueUse);
2964             GPRTemporary scratch(this);
2965             GPRReg scratchReg = scratch.gpr();
2966             m_jit.move(valueOp.gpr(), scratchReg);
2967             if (isClamped) {
2968                 MacroAssembler::Jump inBounds = m_jit.branch64(
2969                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2970                 MacroAssembler::Jump tooBig = m_jit.branch64(
2971                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2972                 m_jit.move(TrustedImm32(0), scratchReg);
2973                 MacroAssembler::Jump clamped = m_jit.jump();
2974                 tooBig.link(&m_jit);
2975                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2976                 clamped.link(&m_jit);
2977                 inBounds.link(&m_jit);
2978             }
2979             value.adopt(scratch);
2980             break;
2981         }
2982 #endif // USE(JSVALUE64)
2983             
2984         case DoubleRepUse: {
2985             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
2986             if (isClamped) {
2987                 SpeculateDoubleOperand valueOp(this, valueUse);
2988                 GPRTemporary result(this);
2989                 FPRTemporary floatScratch(this);
2990                 FPRReg fpr = valueOp.fpr();
2991                 GPRReg gpr = result.gpr();
2992                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
2993                 value.adopt(result);
2994             } else {
2995 #if USE(JSVALUE32_64)
2996                 GPRTemporary realPropertyTag(this);
2997                 propertyTag.adopt(realPropertyTag);
2998                 GPRReg propertyTagGPR = propertyTag.gpr();
2999
3000                 GPRTemporary realValueTag(this);
3001                 valueTag.adopt(realValueTag);
3002                 GPRReg valueTagGPR = valueTag.gpr();
3003 #endif
3004                 SpeculateDoubleOperand valueOp(this, valueUse);
3005                 GPRTemporary result(this);
3006                 FPRReg fpr = valueOp.fpr();
3007                 GPRReg gpr = result.gpr();
3008                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3009                 m_jit.xorPtr(gpr, gpr);
3010                 MacroAssembler::JumpList fixed(m_jit.jump());
3011                 notNaN.link(&m_jit);
3012
3013                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3014                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3015
3016 #if USE(JSVALUE64)
3017                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3018                 boxDouble(fpr, gpr);
3019 #else
3020                 UNUSED_PARAM(property);
3021                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3022                 boxDouble(fpr, valueTagGPR, gpr);
3023 #endif
3024                 slowPathCases.append(m_jit.jump());
3025
3026                 fixed.link(&m_jit);
3027                 value.adopt(result);
3028             }
3029             break;
3030         }
3031             
3032         default:
3033             RELEASE_ASSERT_NOT_REACHED();
3034             break;
3035         }
3036     }
3037     return true;
3038 }
3039
3040 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3041 {
3042     ASSERT(isInt(type));
3043     
3044     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3045     GPRReg storageReg = storage.gpr();
3046     
3047     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3048     
3049     GPRTemporary value;
3050 #if USE(JSVALUE32_64)
3051     GPRTemporary propertyTag;
3052     GPRTemporary valueTag;
3053 #endif
3054
3055     JITCompiler::JumpList slowPathCases;
3056     
3057     bool result = getIntTypedArrayStoreOperand(
3058         value, property,
3059 #if USE(JSVALUE32_64)
3060         propertyTag, valueTag,
3061 #endif
3062         valueUse, slowPathCases, isClamped(type));
3063     if (!result) {
3064         noResult(node);
3065         return;
3066     }
3067
3068     GPRReg valueGPR = value.gpr();
3069 #if USE(JSVALUE32_64)
3070     GPRReg propertyTagGPR = propertyTag.gpr();
3071     GPRReg valueTagGPR = valueTag.gpr();
3072 #endif
3073
3074     ASSERT_UNUSED(valueGPR, valueGPR != property);
3075     ASSERT(valueGPR != base);
3076     ASSERT(valueGPR != storageReg);
3077     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3078
3079     switch (elementSize(type)) {
3080     case 1:
3081         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3082         break;
3083     case 2:
3084         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3085         break;
3086     case 4:
3087         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3088         break;
3089     default:
3090         CRASH();
3091     }
3092
3093     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3094     if (done.isSet())
3095         done.link(&m_jit);
3096
3097     if (!slowPathCases.empty()) {
3098 #if USE(JSVALUE64)
3099         if (node->op() == PutByValDirect) {
3100             addSlowPathGenerator(slowPathCall(
3101                 slowPathCases, this,
3102                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3103                 NoResult, base, property, valueGPR));
3104         } else {
3105             addSlowPathGenerator(slowPathCall(
3106                 slowPathCases, this,
3107                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3108                 NoResult, base, property, valueGPR));
3109         }
3110 #else // not USE(JSVALUE64)
3111         if (node->op() == PutByValDirect) {
3112             addSlowPathGenerator(slowPathCall(
3113                 slowPathCases, this,
3114                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3115                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3116         } else {
3117             addSlowPathGenerator(slowPathCall(
3118                 slowPathCases, this,
3119                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3120                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3121         }
3122 #endif
3123     }
3124     
3125     noResult(node);
3126 }
3127
3128 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3129 {
3130     ASSERT(isFloat(type));
3131     
3132     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3133     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3134     StorageOperand storage(this, m_graph.varArgChild(node, 2));
3135
3136     GPRReg baseReg = base.gpr();
3137     GPRReg propertyReg = property.gpr();
3138     GPRReg storageReg = storage.gpr();
3139
3140     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3141
3142     FPRTemporary result(this);
3143     FPRReg resultReg = result.fpr();
3144     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3145     if (m_indexMaskingMode == IndexMaskingEnabled)
3146         m_jit.and32(MacroAssembler::Address(baseReg, JSObject::butterflyIndexingMaskOffset()), propertyReg);
3147     switch (elementSize(type)) {
3148     case 4:
3149         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3150         m_jit.convertFloatToDouble(resultReg, resultReg);
3151         break;
3152     case 8: {
3153         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3154         break;
3155     }
3156     default:
3157         RELEASE_ASSERT_NOT_REACHED();
3158     }
3159     
3160     doubleResult(resultReg, node);
3161 }
3162
3163 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3164 {
3165     ASSERT(isFloat(type));
3166     
3167     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3168     GPRReg storageReg = storage.gpr();
3169     
3170     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3171     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3172
3173     SpeculateDoubleOperand valueOp(this, valueUse);
3174     FPRTemporary scratch(this);
3175     FPRReg valueFPR = valueOp.fpr();
3176     FPRReg scratchFPR = scratch.fpr();
3177
3178     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3179     
3180     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3181     
3182     switch (elementSize(type)) {
3183     case 4: {
3184         m_jit.moveDouble(valueFPR, scratchFPR);
3185         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3186         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3187         break;
3188     }
3189     case 8:
3190         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3191         break;
3192     default:
3193         RELEASE_ASSERT_NOT_REACHED();
3194     }
3195
3196     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3197     if (done.isSet())
3198         done.link(&m_jit);
3199     noResult(node);
3200 }
3201
3202 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3203 {
3204     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3205     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3206
3207     GPRReg arg1GPR = arg1.gpr();
3208     GPRReg arg2GPR = arg2.gpr();
3209
3210     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3211     speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3212
3213     flushRegisters();
3214     JSValueRegsFlushedCallResult result(this);
3215     JSValueRegs resultRegs = result.regs();
3216     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3217     m_jit.exceptionCheck();
3218
3219     jsValueResult(resultRegs, node);
3220 }
3221
3222 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3223 {
3224     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3225     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3226
3227     GPRReg arg1GPR = arg1.gpr();
3228     GPRReg arg2GPR = arg2.gpr();
3229
3230     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3231     speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3232
3233     flushRegisters();
3234     JSValueRegsFlushedCallResult result(this);
3235     JSValueRegs resultRegs = result.regs();
3236     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3237     m_jit.exceptionCheck();
3238
3239     jsValueResult(resultRegs, node);
3240 }
3241
3242 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3243 {
3244     SpeculateCellOperand arg1(this, child1);
3245     SpeculateCellOperand arg2(this, child2);
3246     JSValueOperand arg3(this, child3);
3247
3248     GPRReg arg1GPR = arg1.gpr();
3249     GPRReg arg2GPR = arg2.gpr();
3250     JSValueRegs arg3Regs = arg3.jsValueRegs();
3251
3252     speculateString(child2, arg2GPR);
3253
3254     flushRegisters();
3255     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3256     m_jit.exceptionCheck();
3257
3258     noResult(node);
3259 }
3260
3261 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3262 {
3263     SpeculateCellOperand arg1(this, child1);
3264     SpeculateCellOperand arg2(this, child2);
3265     JSValueOperand arg3(this, child3);
3266
3267     GPRReg arg1GPR = arg1.gpr();
3268     GPRReg arg2GPR = arg2.gpr();
3269     JSValueRegs arg3Regs = arg3.jsValueRegs();
3270
3271     speculateSymbol(child2, arg2GPR);
3272
3273     flushRegisters();
3274     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3275     m_jit.exceptionCheck();
3276
3277     noResult(node);
3278 }
3279
3280 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg, GPRReg scratch3Reg)
3281 {
3282     // Check that prototype is an object.
3283     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
3284     
3285     // Initialize scratchReg with the value being checked.
3286     m_jit.move(valueReg, scratchReg);
3287     
3288     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
3289     MacroAssembler::Label loop(&m_jit);
3290     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
3291         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
3292     m_jit.emitLoadStructure(*m_jit.vm(), scratchReg, scratch3Reg, scratch2Reg);
3293 #if USE(JSVALUE64)
3294     m_jit.load64(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset()), scratch3Reg);
3295     auto hasMonoProto = m_jit.branchTest64(JITCompiler::NonZero, scratch3Reg);
3296     m_jit.load64(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset)), scratch3Reg);
3297     hasMonoProto.link(&m_jit);
3298     m_jit.move(scratch3Reg, scratchReg);
3299 #else
3300     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + TagOffset), scratch2Reg);
3301     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + PayloadOffset), scratch3Reg);
3302     auto hasMonoProto = m_jit.branch32(CCallHelpers::NotEqual, scratch2Reg, TrustedImm32(JSValue::EmptyValueTag));
3303     m_jit.load32(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), scratch3Reg);
3304     hasMonoProto.link(&m_jit);
3305     m_jit.move(scratch3Reg, scratchReg);
3306 #endif
3307
3308     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
3309 #if USE(JSVALUE64)
3310     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
3311 #else
3312     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
3313 #endif
3314     
3315     // No match - result is false.
3316 #if USE(JSVALUE64)
3317     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
3318 #else
3319     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
3320 #endif
3321     MacroAssembler::JumpList doneJumps; 
3322     doneJumps.append(m_jit.jump());
3323
3324     performDefaultHasInstance.link(&m_jit);
3325     silentSpillAllRegisters(scratchReg);
3326     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg); 
3327     silentFillAllRegisters();
3328     m_jit.exceptionCheck();
3329 #if USE(JSVALUE64)
3330     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
3331 #endif
3332     doneJumps.append(m_jit.jump());
3333     
3334     isInstance.link(&m_jit);
3335 #if USE(JSVALUE64)
3336     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), scratchReg);
3337 #else
3338     m_jit.move(MacroAssembler::TrustedImm32(1), scratchReg);
3339 #endif
3340     
3341     doneJumps.link(&m_jit);
3342 }
3343
3344 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3345 {
3346     SpeculateCellOperand base(this, node->child1());
3347
3348     GPRReg baseGPR = base.gpr();
3349
3350     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3351
3352     noResult(node);
3353 }
3354
3355 void SpeculativeJIT::compileParseInt(Node* node)
3356 {
3357     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3358     if (node->child2()) {
3359         SpeculateInt32Operand radix(this, node->child2());
3360         GPRReg radixGPR = radix.gpr();
3361         if (node->child1().useKind() == UntypedUse) {
3362             JSValueOperand value(this, node->child1());
3363             JSValueRegs valueRegs = value.jsValueRegs();
3364
3365             flushRegisters();
3366             JSValueRegsFlushedCallResult result(this);
3367             JSValueRegs resultRegs = result.regs();
3368             callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3369             m_jit.exceptionCheck();
3370             jsValueResult(resultRegs, node);
3371             return;
3372         }
3373
3374         SpeculateCellOperand value(this, node->child1());
3375         GPRReg valueGPR = value.gpr();
3376         speculateString(node->child1(), valueGPR);
3377
3378         flushRegisters();
3379         JSValueRegsFlushedCallResult result(this);
3380         JSValueRegs resultRegs = result.regs();
3381         callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3382         m_jit.exceptionCheck();
3383         jsValueResult(resultRegs, node);
3384         return;
3385     }
3386
3387     if (node->child1().useKind() == UntypedUse) {
3388         JSValueOperand value(this, node->child1());
3389         JSValueRegs valueRegs = value.jsValueRegs();
3390
3391         flushRegisters();
3392         JSValueRegsFlushedCallResult result(this);
3393         JSValueRegs resultRegs = result.regs();
3394         callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3395         m_jit.exceptionCheck();
3396         jsValueResult(resultRegs, node);
3397         return;
3398     }
3399
3400     SpeculateCellOperand value(this, node->child1());
3401     GPRReg valueGPR = value.gpr();
3402     speculateString(node->child1(), valueGPR);
3403
3404     flushRegisters();
3405     JSValueRegsFlushedCallResult result(this);
3406     JSValueRegs resultRegs = result.regs();
3407     callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3408     m_jit.exceptionCheck();
3409     jsValueResult(resultRegs, node);
3410 }
3411
3412 void SpeculativeJIT::compileInstanceOf(Node* node)
3413 {
3414     if (node->child1().useKind() == UntypedUse) {
3415         // It might not be a cell. Speculate less aggressively.
3416         // Or: it might only be used once (i.e. by us), so we get zero benefit
3417         // from speculating any more aggressively than we absolutely need to.
3418         
3419         JSValueOperand value(this, node->child1());
3420         SpeculateCellOperand prototype(this, node->child2());
3421         GPRTemporary scratch(this);
3422         GPRTemporary scratch2(this);
3423         GPRTemporary scratch3(this);
3424         
3425         GPRReg prototypeReg = prototype.gpr();
3426         GPRReg scratchReg = scratch.gpr();
3427         GPRReg scratch2Reg = scratch2.gpr();
3428         GPRReg scratch3Reg = scratch3.gpr();
3429         
3430         MacroAssembler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs());
3431         GPRReg valueReg = value.jsValueRegs().payloadGPR();
3432         moveFalseTo(scratchReg);
3433
3434         MacroAssembler::Jump done = m_jit.jump();
3435         
3436         isCell.link(&m_jit);
3437         
3438         compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3439         
3440         done.link(&m_jit);
3441
3442         blessedBooleanResult(scratchReg, node);
3443         return;
3444     }
3445     
3446     SpeculateCellOperand value(this, node->child1());
3447     SpeculateCellOperand prototype(this, node->child2());
3448     
3449     GPRTemporary scratch(this);
3450     GPRTemporary scratch2(this);
3451     GPRTemporary scratch3(this);
3452     
3453     GPRReg valueReg = value.gpr();
3454     GPRReg prototypeReg = prototype.gpr();
3455     GPRReg scratchReg = scratch.gpr();
3456     GPRReg scratch2Reg = scratch2.gpr();
3457     GPRReg scratch3Reg = scratch3.gpr();
3458     
3459     compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg, scratch3Reg);
3460
3461     blessedBooleanResult(scratchReg, node);
3462 }
3463
3464 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3465 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3466 {
3467     Edge& leftChild = node->child1();
3468     Edge& rightChild = node->child2();
3469
3470     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3471         JSValueOperand left(this, leftChild);
3472         JSValueOperand right(this, rightChild);
3473         JSValueRegs leftRegs = left.jsValueRegs();
3474         JSValueRegs rightRegs = right.jsValueRegs();
3475
3476         flushRegisters();
3477         JSValueRegsFlushedCallResult result(this);
3478         JSValueRegs resultRegs = result.regs();
3479         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3480         m_jit.exceptionCheck();
3481
3482         jsValueResult(resultRegs, node);
3483         return;
3484     }
3485
3486     std::optional<JSValueOperand> left;
3487     std::optional<JSValueOperand> right;
3488
3489     JSValueRegs leftRegs;
3490     JSValueRegs rightRegs;
3491
3492 #if USE(JSVALUE64)
3493     GPRTemporary result(this);
3494     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3495     GPRTemporary scratch(this);
3496     GPRReg scratchGPR = scratch.gpr();
3497 #else
3498     GPRTemporary resultTag(this);
3499     GPRTemporary resultPayload(this);
3500     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3501     GPRReg scratchGPR = resultTag.gpr();
3502 #endif
3503
3504     SnippetOperand leftOperand;
3505     SnippetOperand rightOperand;
3506
3507     // The snippet generator does not support both operands being constant. If the left
3508     // operand is already const, we'll ignore the right operand's constness.
3509     if (leftChild->isInt32Constant())
3510         leftOperand.setConstInt32(leftChild->asInt32());
3511     else if (rightChild->isInt32Constant())
3512         rightOperand.setConstInt32(rightChild->asInt32());
3513
3514     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3515
3516     if (!leftOperand.isConst()) {
3517         left.emplace(this, leftChild);
3518         leftRegs = left->jsValueRegs();
3519     }
3520     if (!rightOperand.isConst()) {
3521         right.emplace(this, rightChild);
3522         rightRegs = right->jsValueRegs();
3523     }
3524
3525     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3526     gen.generateFastPath(m_jit);
3527
3528     ASSERT(gen.didEmitFastPath());
3529     gen.endJumpList().append(m_jit.jump());
3530
3531     gen.slowPathJumpList().link(&m_jit);
3532     silentSpillAllRegisters(resultRegs);
3533
3534     if (leftOperand.isConst()) {
3535         leftRegs = resultRegs;
3536         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3537     } else if (rightOperand.isConst()) {
3538         rightRegs = resultRegs;
3539         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3540     }
3541
3542     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3543
3544     silentFillAllRegisters();
3545     m_jit.exceptionCheck();
3546
3547     gen.endJumpList().link(&m_jit);
3548     jsValueResult(resultRegs, node);
3549 }
3550
3551 void SpeculativeJIT::compileBitwiseOp(Node* node)
3552 {
3553     NodeType op = node->op();
3554     Edge& leftChild = node->child1();
3555     Edge& rightChild = node->child2();
3556
3557     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3558         switch (op) {
3559         case BitAnd:
3560             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3561             return;
3562         case BitOr:
3563             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3564             return;
3565         case BitXor:
3566             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3567             return;
3568         default:
3569             RELEASE_ASSERT_NOT_REACHED();
3570         }
3571     }
3572
3573     if (leftChild->isInt32Constant()) {
3574         SpeculateInt32Operand op2(this, rightChild);
3575         GPRTemporary result(this, Reuse, op2);
3576
3577         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3578
3579         int32Result(result.gpr(), node);
3580
3581     } else if (rightChild->isInt32Constant()) {
3582         SpeculateInt32Operand op1(this, leftChild);
3583         GPRTemporary result(this, Reuse, op1);
3584
3585         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3586
3587         int32Result(result.gpr(), node);
3588
3589     } else {
3590         SpeculateInt32Operand op1(this, leftChild);
3591         SpeculateInt32Operand op2(this, rightChild);
3592         GPRTemporary result(this, Reuse, op1, op2);
3593         
3594         GPRReg reg1 = op1.gpr();
3595         GPRReg reg2 = op2.gpr();
3596         bitOp(op, reg1, reg2, result.gpr());
3597         
3598         int32Result(result.gpr(), node);
3599     }
3600 }
3601
3602 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3603 {
3604     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3605         ? operationValueBitRShift : operationValueBitURShift;
3606     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3607         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3608
3609     Edge& leftChild = node->child1();
3610     Edge& rightChild = node->child2();
3611
3612     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3613         JSValueOperand left(this, leftChild);
3614         JSValueOperand right(this, rightChild);
3615         JSValueRegs leftRegs = left.jsValueRegs();
3616         JSValueRegs rightRegs = right.jsValueRegs();
3617
3618         flushRegisters();
3619         JSValueRegsFlushedCallResult result(this);
3620         JSValueRegs resultRegs = result.regs();
3621         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3622         m_jit.exceptionCheck();
3623
3624         jsValueResult(resultRegs, node);
3625         return;
3626     }
3627
3628     std::optional<JSValueOperand> left;
3629     std::optional<JSValueOperand> right;
3630
3631     JSValueRegs leftRegs;
3632     JSValueRegs rightRegs;
3633
3634     FPRTemporary leftNumber(this);
3635     FPRReg leftFPR = leftNumber.fpr();
3636
3637 #if USE(JSVALUE64)
3638     GPRTemporary result(this);
3639     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3640     GPRTemporary scratch(this);
3641     GPRReg scratchGPR = scratch.gpr();
3642     FPRReg scratchFPR = InvalidFPRReg;
3643 #else
3644     GPRTemporary resultTag(this);
3645     GPRTemporary resultPayload(this);
3646     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3647     GPRReg scratchGPR = resultTag.gpr();
3648     FPRTemporary fprScratch(this);
3649     FPRReg scratchFPR = fprScratch.fpr();
3650 #endif
3651
3652     SnippetOperand leftOperand;
3653     SnippetOperand rightOperand;
3654
3655     // The snippet generator does not support both operands being constant. If the left
3656     // operand is already const, we'll ignore the right operand's constness.
3657     if (leftChild->isInt32Constant())
3658         leftOperand.setConstInt32(leftChild->asInt32());
3659     else if (rightChild->isInt32Constant())
3660         rightOperand.setConstInt32(rightChild->asInt32());
3661
3662     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3663
3664     if (!leftOperand.isConst()) {
3665         left.emplace(this, leftChild);
3666         leftRegs = left->jsValueRegs();
3667     }
3668     if (!rightOperand.isConst()) {
3669         right.emplace(this, rightChild);
3670         rightRegs = right->jsValueRegs();
3671     }
3672
3673     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3674         leftFPR, scratchGPR, scratchFPR, shiftType);
3675     gen.generateFastPath(m_jit);
3676
3677     ASSERT(gen.didEmitFastPath());
3678     gen.endJumpList().append(m_jit.jump());
3679
3680     gen.slowPathJumpList().link(&m_jit);
3681     silentSpillAllRegisters(resultRegs);
3682
3683     if (leftOperand.isConst()) {
3684         leftRegs = resultRegs;
3685         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3686     } else if (rightOperand.isConst()) {
3687         rightRegs = resultRegs;
3688         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3689     }
3690
3691     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3692
3693     silentFillAllRegisters();
3694     m_jit.exceptionCheck();
3695
3696     gen.endJumpList().link(&m_jit);
3697     jsValueResult(resultRegs, node);
3698     return;
3699 }
3700
3701 void SpeculativeJIT::compileShiftOp(Node* node)
3702 {
3703     NodeType op = node->op();
3704     Edge& leftChild = node->child1();
3705     Edge& rightChild = node->child2();
3706
3707     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3708         switch (op) {
3709         case BitLShift:
3710             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3711             return;
3712         case BitRShift:
3713         case BitURShift:
3714             emitUntypedRightShiftBitOp(node);
3715             return;
3716         default:
3717             RELEASE_ASSERT_NOT_REACHED();
3718         }
3719     }
3720
3721     if (rightChild->isInt32Constant()) {
3722         SpeculateInt32Operand op1(this, leftChild);
3723         GPRTemporary result(this, Reuse, op1);
3724
3725         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3726
3727         int32Result(result.gpr(), node);
3728     } else {
3729         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3730         SpeculateInt32Operand op1(this, leftChild);
3731         SpeculateInt32Operand op2(this, rightChild);
3732         GPRTemporary result(this, Reuse, op1);
3733
3734         GPRReg reg1 = op1.gpr();
3735         GPRReg reg2 = op2.gpr();
3736         shiftOp(op, reg1, reg2, result.gpr());
3737
3738         int32Result(result.gpr(), node);
3739     }
3740 }
3741
3742 void SpeculativeJIT::compileValueAdd(Node* node)
3743 {
3744     Edge& leftChild = node->child1();
3745     Edge& rightChild = node->child2();
3746
3747     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3748         JSValueOperand left(this, leftChild);
3749         JSValueOperand right(this, rightChild);
3750         JSValueRegs leftRegs = left.jsValueRegs();
3751         JSValueRegs rightRegs = right.jsValueRegs();
3752
3753         flushRegisters();
3754         JSValueRegsFlushedCallResult result(this);
3755         JSValueRegs resultRegs = result.regs();
3756         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3757         m_jit.exceptionCheck();
3758     
3759         jsValueResult(resultRegs, node);
3760         return;
3761     }
3762
3763 #if USE(JSVALUE64)
3764     bool needsScratchGPRReg = true;
3765     bool needsScratchFPRReg = false;
3766 #else
3767     bool needsScratchGPRReg = true;
3768     bool needsScratchFPRReg = true;
3769 #endif
3770
3771     ArithProfile* arithProfile = m_jit.graph().baselineCodeBlockFor(node->origin.semantic)->arithProfileForBytecodeOffset(node->origin.semantic.bytecodeIndex);
3772     JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile);
3773     auto repatchingFunction = operationValueAddOptimize;
3774     auto nonRepatchingFunction = operationValueAdd;
3775     
3776     compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3777 }
3778
3779 template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3780 void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3781 {
3782     Edge& leftChild = node->child1();
3783     Edge& rightChild = node->child2();
3784
3785     std::optional<JSValueOperand> left;
3786     std::optional<JSValueOperand> right;
3787
3788     JSValueRegs leftRegs;
3789     JSValueRegs rightRegs;
3790
3791     FPRTemporary leftNumber(this);
3792     FPRTemporary rightNumber(this);
3793     FPRReg leftFPR = leftNumber.fpr();
3794     FPRReg rightFPR = rightNumber.fpr();
3795
3796     GPRReg scratchGPR = InvalidGPRReg;
3797     FPRReg scratchFPR = InvalidFPRReg;
3798
3799     std::optional<FPRTemporary> fprScratch;
3800     if (needsScratchFPRReg) {
3801         fprScratch.emplace(this);
3802         scratchFPR = fprScratch->fpr();
3803     }
3804
3805 #if USE(JSVALUE64)
3806     std::optional<GPRTemporary> gprScratch;
3807     if (needsScratchGPRReg) {
3808         gprScratch.emplace(this);
3809         scratchGPR = gprScratch->gpr();
3810     }
3811     GPRTemporary result(this);
3812     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3813 #else
3814     GPRTemporary resultTag(this);
3815     GPRTemporary resultPayload(this);
3816     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3817     if (needsScratchGPRReg)
3818         scratchGPR = resultRegs.tagGPR();
3819 #endif
3820
3821     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3822     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3823
3824     // The snippet generator does not support both operands being constant. If the left
3825     // operand is already const, we'll ignore the right operand's constness.
3826     if (leftChild->isInt32Constant())
3827         leftOperand.setConstInt32(leftChild->asInt32());
3828     else if (rightChild->isInt32Constant())
3829         rightOperand.setConstInt32(rightChild->asInt32());
3830
3831     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3832     ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
3833
3834     if (!Generator::isLeftOperandValidConstant(leftOperand)) {
3835         left.emplace(this, leftChild);
3836         leftRegs = left->jsValueRegs();
3837     }
3838     if (!Generator::isRightOperandValidConstant(rightOperand)) {
3839         right.emplace(this, rightChild);
3840         rightRegs = right->jsValueRegs();
3841     }
3842
3843 #if ENABLE(MATH_IC_STATS)
3844     auto inlineStart = m_jit.label();
3845 #endif
3846
3847     Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
3848     mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
3849
3850     bool shouldEmitProfiling = false;
3851     bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
3852
3853     if (generatedInline) {
3854         ASSERT(!addICGenerationState->slowPathJumps.empty());
3855
3856         Vector<SilentRegisterSavePlan> savePlans;
3857         silentSpillAllRegistersImpl(false, savePlans, resultRegs);
3858
3859         auto done = m_jit.label();
3860
3861         addSlowPathGenerator([=, savePlans = WTFMove(savePlans)] () {
3862             addICGenerationState->slowPathJumps.link(&m_jit);
3863             addICGenerationState->slowPathStart = m_jit.label();
3864 #if ENABLE(MATH_IC_STATS)
3865             auto slowPathStart = m_jit.label();
3866 #endif
3867
3868             silentSpill(savePlans);
3869
3870             auto innerLeftRegs = leftRegs;
3871             auto innerRightRegs = rightRegs;
3872             if (Generator::isLeftOperandValidConstant(leftOperand)) {
3873                 innerLeftRegs = resultRegs;
3874                 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
3875             } else if (Generator::isRightOperandValidConstant(rightOperand)) {
3876                 innerRightRegs = resultRegs;
3877                 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
3878             }
3879
3880             if (addICGenerationState->shouldSlowPathRepatch)
3881                 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
3882             else
3883                 addICGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, innerLeftRegs, innerRightRegs);
3884
3885             silentFill(savePlans);
3886             m_jit.exceptionCheck();
3887             m_jit.jump().linkTo(done, &m_jit);
3888
3889             m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
3890                 mathIC->finalizeInlineCode(*addICGenerationState, linkBuffer);
3891             });
3892
3893 #if ENABLE(MATH_IC_STATS)