fe8be374a0fbf0c0140df2318010090496b1b520
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGCapabilities.h"
37 #include "DFGMayExit.h"
38 #include "DFGOSRExitFuzz.h"
39 #include "DFGSaneStringGetByValSlowPathGenerator.h"
40 #include "DFGSlowPathGenerator.h"
41 #include "DFGSnippetParams.h"
42 #include "DirectArguments.h"
43 #include "JITAddGenerator.h"
44 #include "JITBitAndGenerator.h"
45 #include "JITBitOrGenerator.h"
46 #include "JITBitXorGenerator.h"
47 #include "JITDivGenerator.h"
48 #include "JITLeftShiftGenerator.h"
49 #include "JITMulGenerator.h"
50 #include "JITRightShiftGenerator.h"
51 #include "JITSubGenerator.h"
52 #include "JSAsyncFunction.h"
53 #include "JSAsyncGeneratorFunction.h"
54 #include "JSCInlines.h"
55 #include "JSFixedArray.h"
56 #include "JSGeneratorFunction.h"
57 #include "JSLexicalEnvironment.h"
58 #include "JSPropertyNameEnumerator.h"
59 #include "LinkBuffer.h"
60 #include "RegExpConstructor.h"
61 #include "ScopedArguments.h"
62 #include "ScratchRegisterAllocator.h"
63 #include "SuperSampler.h"
64 #include "TypeProfilerLog.h"
65 #include "WeakMapImpl.h"
66 #include <wtf/BitVector.h>
67 #include <wtf/Box.h>
68 #include <wtf/MathExtras.h>
69
70 namespace JSC { namespace DFG {
71
72 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
73     : m_compileOkay(true)
74     , m_jit(jit)
75     , m_graph(m_jit.graph())
76     , m_currentNode(0)
77     , m_lastGeneratedNode(LastNodeType)
78     , m_indexInBlock(0)
79     , m_generationInfo(m_jit.graph().frameRegisterCount())
80     , m_state(m_jit.graph())
81     , m_interpreter(m_jit.graph(), m_state)
82     , m_stream(&jit.jitCode()->variableEventStream)
83     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
84 {
85 }
86
87 SpeculativeJIT::~SpeculativeJIT()
88 {
89 }
90
91 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
92 {
93     IndexingType indexingType = structure->indexingType();
94     bool hasIndexingHeader = hasIndexedProperties(indexingType);
95
96     unsigned inlineCapacity = structure->inlineCapacity();
97     unsigned outOfLineCapacity = structure->outOfLineCapacity();
98     
99     GPRTemporary scratch(this);
100     GPRTemporary scratch2(this);
101     GPRReg scratchGPR = scratch.gpr();
102     GPRReg scratch2GPR = scratch2.gpr();
103
104     ASSERT(vectorLength >= numElements);
105     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
106     
107     JITCompiler::JumpList slowCases;
108
109     size_t size = 0;
110     if (hasIndexingHeader)
111         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
112     size += outOfLineCapacity * sizeof(JSValue);
113
114     m_jit.move(TrustedImmPtr(nullptr), storageGPR);
115
116     if (size) {
117         if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
118             m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
119             
120             m_jit.addPtr(
121                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
122                 storageGPR);
123             
124             if (hasIndexingHeader)
125                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
126         } else
127             slowCases.append(m_jit.jump());
128     }
129
130     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
131     Allocator allocator = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
132     if (allocator) {
133         emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
134         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
135     } else
136         slowCases.append(m_jit.jump());
137
138     // I want a slow path that also loads out the storage pointer, and that's
139     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
140     // of work for a very small piece of functionality. :-/
141     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
142         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
143         structure, vectorLength));
144
145     if (numElements < vectorLength) {
146 #if USE(JSVALUE64)
147         if (hasDouble(structure->indexingType()))
148             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
149         else
150             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
151         for (unsigned i = numElements; i < vectorLength; ++i)
152             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
153 #else
154         EncodedValueDescriptor value;
155         if (hasDouble(structure->indexingType()))
156             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
157         else
158             value.asInt64 = JSValue::encode(JSValue());
159         for (unsigned i = numElements; i < vectorLength; ++i) {
160             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
161             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
162         }
163 #endif
164     }
165     
166     if (hasIndexingHeader)
167         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
168     
169     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
170     
171     m_jit.mutatorFence(*m_jit.vm());
172 }
173
174 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
175 {
176     if (inlineCallFrame && !inlineCallFrame->isVarargs())
177         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
178     else {
179         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
180         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
181         if (!includeThis)
182             m_jit.sub32(TrustedImm32(1), lengthGPR);
183     }
184 }
185
186 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
187 {
188     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
189 }
190
191 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
192 {
193     if (origin.inlineCallFrame) {
194         if (origin.inlineCallFrame->isClosureCall) {
195             m_jit.loadPtr(
196                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
197                 calleeGPR);
198         } else {
199             m_jit.move(
200                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
201                 calleeGPR);
202         }
203     } else
204         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
205 }
206
207 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
208 {
209     m_jit.addPtr(
210         TrustedImm32(
211             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
212         GPRInfo::callFrameRegister, startGPR);
213 }
214
215 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
216 {
217     if (!Options::useOSRExitFuzz()
218         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
219         || !doOSRExitFuzzing())
220         return MacroAssembler::Jump();
221     
222     MacroAssembler::Jump result;
223     
224     m_jit.pushToSave(GPRInfo::regT0);
225     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
226     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
227     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
228     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
229     unsigned at = Options::fireOSRExitFuzzAt();
230     if (at || atOrAfter) {
231         unsigned threshold;
232         MacroAssembler::RelationalCondition condition;
233         if (atOrAfter) {
234             threshold = atOrAfter;
235             condition = MacroAssembler::Below;
236         } else {
237             threshold = at;
238             condition = MacroAssembler::NotEqual;
239         }
240         MacroAssembler::Jump ok = m_jit.branch32(
241             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
242         m_jit.popToRestore(GPRInfo::regT0);
243         result = m_jit.jump();
244         ok.link(&m_jit);
245     }
246     m_jit.popToRestore(GPRInfo::regT0);
247     
248     return result;
249 }
250
251 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
252 {
253     if (!m_compileOkay)
254         return;
255     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
256     if (fuzzJump.isSet()) {
257         JITCompiler::JumpList jumpsToFail;
258         jumpsToFail.append(fuzzJump);
259         jumpsToFail.append(jumpToFail);
260         m_jit.appendExitInfo(jumpsToFail);
261     } else
262         m_jit.appendExitInfo(jumpToFail);
263     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
264 }
265
266 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
267 {
268     if (!m_compileOkay)
269         return;
270     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
271     if (fuzzJump.isSet()) {
272         JITCompiler::JumpList myJumpsToFail;
273         myJumpsToFail.append(jumpsToFail);
274         myJumpsToFail.append(fuzzJump);
275         m_jit.appendExitInfo(myJumpsToFail);
276     } else
277         m_jit.appendExitInfo(jumpsToFail);
278     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
279 }
280
281 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
282 {
283     if (!m_compileOkay)
284         return OSRExitJumpPlaceholder();
285     unsigned index = m_jit.jitCode()->osrExit.size();
286     m_jit.appendExitInfo();
287     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
288     return OSRExitJumpPlaceholder(index);
289 }
290
291 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
292 {
293     return speculationCheck(kind, jsValueSource, nodeUse.node());
294 }
295
296 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
297 {
298     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
299 }
300
301 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
302 {
303     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
304 }
305
306 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
307 {
308     if (!m_compileOkay)
309         return;
310     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
311     m_jit.appendExitInfo(jumpToFail);
312     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
313 }
314
315 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
316 {
317     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
318 }
319
320 void SpeculativeJIT::emitInvalidationPoint(Node* node)
321 {
322     if (!m_compileOkay)
323         return;
324     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
325     m_jit.jitCode()->appendOSRExit(OSRExit(
326         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
327         this, m_stream->size()));
328     info.m_replacementSource = m_jit.watchpointLabel();
329     ASSERT(info.m_replacementSource.isSet());
330     noResult(node);
331 }
332
333 void SpeculativeJIT::unreachable(Node* node)
334 {
335     m_compileOkay = false;
336     m_jit.abortWithReason(DFGUnreachableNode, node->op());
337 }
338
339 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
340 {
341     if (!m_compileOkay)
342         return;
343     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
344     m_compileOkay = false;
345     if (verboseCompilationEnabled())
346         dataLog("Bailing compilation.\n");
347 }
348
349 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
350 {
351     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
352 }
353
354 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
355 {
356     ASSERT(needsTypeCheck(edge, typesPassedThrough));
357     m_interpreter.filter(edge, typesPassedThrough);
358     speculationCheck(exitKind, source, edge.node(), jumpToFail);
359 }
360
361 RegisterSet SpeculativeJIT::usedRegisters()
362 {
363     RegisterSet result;
364     
365     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
366         GPRReg gpr = GPRInfo::toRegister(i);
367         if (m_gprs.isInUse(gpr))
368             result.set(gpr);
369     }
370     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
371         FPRReg fpr = FPRInfo::toRegister(i);
372         if (m_fprs.isInUse(fpr))
373             result.set(fpr);
374     }
375     
376     // FIXME: This is overly conservative. We could subtract out those callee-saves that we
377     // actually saved.
378     // https://bugs.webkit.org/show_bug.cgi?id=185686
379     result.merge(RegisterSet::stubUnavailableRegisters());
380     
381     return result;
382 }
383
384 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
385 {
386     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
387 }
388
389 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
390 {
391     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
392 }
393
394 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
395 {
396     for (auto& slowPathGenerator : m_slowPathGenerators) {
397         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
398         slowPathGenerator->generate(this);
399     }
400     for (auto& slowPathLambda : m_slowPathLambdas) {
401         Node* currentNode = slowPathLambda.currentNode;
402         m_currentNode = currentNode;
403         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
404         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
405         slowPathLambda.generator();
406         m_outOfLineStreamIndex = std::nullopt;
407     }
408 }
409
410 void SpeculativeJIT::clearGenerationInfo()
411 {
412     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
413         m_generationInfo[i] = GenerationInfo();
414     m_gprs = RegisterBank<GPRInfo>();
415     m_fprs = RegisterBank<FPRInfo>();
416 }
417
418 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
419 {
420     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
421     Node* node = info.node();
422     DataFormat registerFormat = info.registerFormat();
423     ASSERT(registerFormat != DataFormatNone);
424     ASSERT(registerFormat != DataFormatDouble);
425         
426     SilentSpillAction spillAction;
427     SilentFillAction fillAction;
428         
429     if (!info.needsSpill())
430         spillAction = DoNothingForSpill;
431     else {
432 #if USE(JSVALUE64)
433         ASSERT(info.gpr() == source);
434         if (registerFormat == DataFormatInt32)
435             spillAction = Store32Payload;
436         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
437             spillAction = StorePtr;
438         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
439             spillAction = Store64;
440         else {
441             ASSERT(registerFormat & DataFormatJS);
442             spillAction = Store64;
443         }
444 #elif USE(JSVALUE32_64)
445         if (registerFormat & DataFormatJS) {
446             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
447             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
448         } else {
449             ASSERT(info.gpr() == source);
450             spillAction = Store32Payload;
451         }
452 #endif
453     }
454         
455     if (registerFormat == DataFormatInt32) {
456         ASSERT(info.gpr() == source);
457         ASSERT(isJSInt32(info.registerFormat()));
458         if (node->hasConstant()) {
459             ASSERT(node->isInt32Constant());
460             fillAction = SetInt32Constant;
461         } else
462             fillAction = Load32Payload;
463     } else if (registerFormat == DataFormatBoolean) {
464 #if USE(JSVALUE64)
465         RELEASE_ASSERT_NOT_REACHED();
466 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
467         fillAction = DoNothingForFill;
468 #endif
469 #elif USE(JSVALUE32_64)
470         ASSERT(info.gpr() == source);
471         if (node->hasConstant()) {
472             ASSERT(node->isBooleanConstant());
473             fillAction = SetBooleanConstant;
474         } else
475             fillAction = Load32Payload;
476 #endif
477     } else if (registerFormat == DataFormatCell) {
478         ASSERT(info.gpr() == source);
479         if (node->hasConstant()) {
480             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
481             node->asCell(); // To get the assertion.
482             fillAction = SetCellConstant;
483         } else {
484 #if USE(JSVALUE64)
485             fillAction = LoadPtr;
486 #else
487             fillAction = Load32Payload;
488 #endif
489         }
490     } else if (registerFormat == DataFormatStorage) {
491         ASSERT(info.gpr() == source);
492         fillAction = LoadPtr;
493     } else if (registerFormat == DataFormatInt52) {
494         if (node->hasConstant())
495             fillAction = SetInt52Constant;
496         else if (info.spillFormat() == DataFormatInt52)
497             fillAction = Load64;
498         else if (info.spillFormat() == DataFormatStrictInt52)
499             fillAction = Load64ShiftInt52Left;
500         else if (info.spillFormat() == DataFormatNone)
501             fillAction = Load64;
502         else {
503             RELEASE_ASSERT_NOT_REACHED();
504 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
505             fillAction = Load64; // Make GCC happy.
506 #endif
507         }
508     } else if (registerFormat == DataFormatStrictInt52) {
509         if (node->hasConstant())
510             fillAction = SetStrictInt52Constant;
511         else if (info.spillFormat() == DataFormatInt52)
512             fillAction = Load64ShiftInt52Right;
513         else if (info.spillFormat() == DataFormatStrictInt52)
514             fillAction = Load64;
515         else if (info.spillFormat() == DataFormatNone)
516             fillAction = Load64;
517         else {
518             RELEASE_ASSERT_NOT_REACHED();
519 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
520             fillAction = Load64; // Make GCC happy.
521 #endif
522         }
523     } else {
524         ASSERT(registerFormat & DataFormatJS);
525 #if USE(JSVALUE64)
526         ASSERT(info.gpr() == source);
527         if (node->hasConstant()) {
528             if (node->isCellConstant())
529                 fillAction = SetTrustedJSConstant;
530             else
531                 fillAction = SetJSConstant;
532         } else if (info.spillFormat() == DataFormatInt32) {
533             ASSERT(registerFormat == DataFormatJSInt32);
534             fillAction = Load32PayloadBoxInt;
535         } else
536             fillAction = Load64;
537 #else
538         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
539         if (node->hasConstant())
540             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
541         else if (info.payloadGPR() == source)
542             fillAction = Load32Payload;
543         else { // Fill the Tag
544             switch (info.spillFormat()) {
545             case DataFormatInt32:
546                 ASSERT(registerFormat == DataFormatJSInt32);
547                 fillAction = SetInt32Tag;
548                 break;
549             case DataFormatCell:
550                 ASSERT(registerFormat == DataFormatJSCell);
551                 fillAction = SetCellTag;
552                 break;
553             case DataFormatBoolean:
554                 ASSERT(registerFormat == DataFormatJSBoolean);
555                 fillAction = SetBooleanTag;
556                 break;
557             default:
558                 fillAction = Load32Tag;
559                 break;
560             }
561         }
562 #endif
563     }
564         
565     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
566 }
567     
568 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
569 {
570     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
571     Node* node = info.node();
572     ASSERT(info.registerFormat() == DataFormatDouble);
573
574     SilentSpillAction spillAction;
575     SilentFillAction fillAction;
576         
577     if (!info.needsSpill())
578         spillAction = DoNothingForSpill;
579     else {
580         ASSERT(!node->hasConstant());
581         ASSERT(info.spillFormat() == DataFormatNone);
582         ASSERT(info.fpr() == source);
583         spillAction = StoreDouble;
584     }
585         
586 #if USE(JSVALUE64)
587     if (node->hasConstant()) {
588         node->asNumber(); // To get the assertion.
589         fillAction = SetDoubleConstant;
590     } else {
591         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
592         fillAction = LoadDouble;
593     }
594 #elif USE(JSVALUE32_64)
595     ASSERT(info.registerFormat() == DataFormatDouble);
596     if (node->hasConstant()) {
597         node->asNumber(); // To get the assertion.
598         fillAction = SetDoubleConstant;
599     } else
600         fillAction = LoadDouble;
601 #endif
602
603     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
604 }
605     
606 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
607 {
608     switch (plan.spillAction()) {
609     case DoNothingForSpill:
610         break;
611     case Store32Tag:
612         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
613         break;
614     case Store32Payload:
615         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
616         break;
617     case StorePtr:
618         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
619         break;
620 #if USE(JSVALUE64)
621     case Store64:
622         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
623         break;
624 #endif
625     case StoreDouble:
626         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
627         break;
628     default:
629         RELEASE_ASSERT_NOT_REACHED();
630     }
631 }
632     
633 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
634 {
635     switch (plan.fillAction()) {
636     case DoNothingForFill:
637         break;
638     case SetInt32Constant:
639         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
640         break;
641 #if USE(JSVALUE64)
642     case SetInt52Constant:
643         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
644         break;
645     case SetStrictInt52Constant:
646         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
647         break;
648 #endif // USE(JSVALUE64)
649     case SetBooleanConstant:
650         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
651         break;
652     case SetCellConstant:
653         ASSERT(plan.node()->constant()->value().isCell());
654         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
655         break;
656 #if USE(JSVALUE64)
657     case SetTrustedJSConstant:
658         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
659         break;
660     case SetJSConstant:
661         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
662         break;
663     case SetDoubleConstant:
664         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
665         break;
666     case Load32PayloadBoxInt:
667         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
668         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
669         break;
670     case Load32PayloadConvertToInt52:
671         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
672         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
673         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
674         break;
675     case Load32PayloadSignExtend:
676         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
677         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
678         break;
679 #else
680     case SetJSConstantTag:
681         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
682         break;
683     case SetJSConstantPayload:
684         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
685         break;
686     case SetInt32Tag:
687         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
688         break;
689     case SetCellTag:
690         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
691         break;
692     case SetBooleanTag:
693         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
694         break;
695     case SetDoubleConstant:
696         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
697         break;
698 #endif
699     case Load32Tag:
700         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
701         break;
702     case Load32Payload:
703         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
704         break;
705     case LoadPtr:
706         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
707         break;
708 #if USE(JSVALUE64)
709     case Load64:
710         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
711         break;
712     case Load64ShiftInt52Right:
713         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
714         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
715         break;
716     case Load64ShiftInt52Left:
717         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
718         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
719         break;
720 #endif
721     case LoadDouble:
722         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
723         break;
724     default:
725         RELEASE_ASSERT_NOT_REACHED();
726     }
727 }
728
729 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
730 {
731     JITCompiler::JumpList result;
732     
733     switch (arrayMode.type()) {
734     case Array::Int32:
735     case Array::Double:
736     case Array::Contiguous:
737     case Array::Undecided:
738     case Array::ArrayStorage: {
739         IndexingType shape = arrayMode.shapeMask();
740         switch (arrayMode.arrayClass()) {
741         case Array::OriginalArray:
742             RELEASE_ASSERT_NOT_REACHED();
743             return result;
744
745         case Array::Array:
746             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
747             result.append(m_jit.branch32(
748                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
749             return result;
750
751         case Array::NonArray:
752         case Array::OriginalNonArray:
753             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
754             result.append(m_jit.branch32(
755                 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
756             return result;
757
758         case Array::PossiblyArray:
759             m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
760             result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
761             return result;
762         }
763
764         RELEASE_ASSERT_NOT_REACHED();
765         return result;
766     }
767
768     case Array::SlowPutArrayStorage: {
769         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
770
771         switch (arrayMode.arrayClass()) {
772         case Array::OriginalArray:
773             RELEASE_ASSERT_NOT_REACHED();
774             return result;
775
776         case Array::Array:
777             result.append(
778                 m_jit.branchTest32(
779                     MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
780             break;
781
782         case Array::NonArray:
783         case Array::OriginalNonArray:
784             result.append(
785                 m_jit.branchTest32(
786                     MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
787             break;
788
789         case Array::PossiblyArray:
790             break;
791         }
792
793         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
794         m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
795         result.append(
796             m_jit.branch32(
797                 MacroAssembler::Above, tempGPR,
798                 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
799         return result;
800     }
801     default:
802         CRASH();
803         break;
804     }
805     
806     return result;
807 }
808
809 void SpeculativeJIT::checkArray(Node* node)
810 {
811     ASSERT(node->arrayMode().isSpecific());
812     ASSERT(!node->arrayMode().doesConversion());
813     
814     SpeculateCellOperand base(this, node->child1());
815     GPRReg baseReg = base.gpr();
816     
817     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
818         noResult(m_currentNode);
819         return;
820     }
821     
822     switch (node->arrayMode().type()) {
823     case Array::AnyTypedArray:
824     case Array::String:
825         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
826         return;
827     case Array::Int32:
828     case Array::Double:
829     case Array::Contiguous:
830     case Array::Undecided:
831     case Array::ArrayStorage:
832     case Array::SlowPutArrayStorage: {
833         GPRTemporary temp(this);
834         GPRReg tempGPR = temp.gpr();
835         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
836         speculationCheck(
837             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
838             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
839         
840         noResult(m_currentNode);
841         return;
842     }
843     case Array::DirectArguments:
844         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
845         noResult(m_currentNode);
846         return;
847     case Array::ScopedArguments:
848         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
849         noResult(m_currentNode);
850         return;
851     default:
852         speculateCellTypeWithoutTypeFiltering(
853             node->child1(), baseReg,
854             typeForTypedArrayType(node->arrayMode().typedArrayType()));
855         noResult(m_currentNode);
856         return;
857     }
858 }
859
860 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
861 {
862     ASSERT(node->arrayMode().doesConversion());
863     
864     GPRTemporary temp(this);
865     GPRTemporary structure;
866     GPRReg tempGPR = temp.gpr();
867     GPRReg structureGPR = InvalidGPRReg;
868     
869     if (node->op() != ArrayifyToStructure) {
870         GPRTemporary realStructure(this);
871         structure.adopt(realStructure);
872         structureGPR = structure.gpr();
873     }
874         
875     // We can skip all that comes next if we already have array storage.
876     MacroAssembler::JumpList slowPath;
877     
878     if (node->op() == ArrayifyToStructure) {
879         slowPath.append(m_jit.branchWeakStructure(
880             JITCompiler::NotEqual,
881             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
882             node->structure()));
883     } else {
884         m_jit.load8(
885             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
886         
887         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
888     }
889     
890     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
891         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
892     
893     noResult(m_currentNode);
894 }
895
896 void SpeculativeJIT::arrayify(Node* node)
897 {
898     ASSERT(node->arrayMode().isSpecific());
899     
900     SpeculateCellOperand base(this, node->child1());
901     
902     if (!node->child2()) {
903         arrayify(node, base.gpr(), InvalidGPRReg);
904         return;
905     }
906     
907     SpeculateInt32Operand property(this, node->child2());
908     
909     arrayify(node, base.gpr(), property.gpr());
910 }
911
912 GPRReg SpeculativeJIT::fillStorage(Edge edge)
913 {
914     VirtualRegister virtualRegister = edge->virtualRegister();
915     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
916     
917     switch (info.registerFormat()) {
918     case DataFormatNone: {
919         if (info.spillFormat() == DataFormatStorage) {
920             GPRReg gpr = allocate();
921             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
922             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
923             info.fillStorage(*m_stream, gpr);
924             return gpr;
925         }
926         
927         // Must be a cell; fill it as a cell and then return the pointer.
928         return fillSpeculateCell(edge);
929     }
930         
931     case DataFormatStorage: {
932         GPRReg gpr = info.gpr();
933         m_gprs.lock(gpr);
934         return gpr;
935     }
936         
937     default:
938         return fillSpeculateCell(edge);
939     }
940 }
941
942 void SpeculativeJIT::useChildren(Node* node)
943 {
944     if (node->flags() & NodeHasVarArgs) {
945         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
946             if (!!m_jit.graph().m_varArgChildren[childIdx])
947                 use(m_jit.graph().m_varArgChildren[childIdx]);
948         }
949     } else {
950         Edge child1 = node->child1();
951         if (!child1) {
952             ASSERT(!node->child2() && !node->child3());
953             return;
954         }
955         use(child1);
956         
957         Edge child2 = node->child2();
958         if (!child2) {
959             ASSERT(!node->child3());
960             return;
961         }
962         use(child2);
963         
964         Edge child3 = node->child3();
965         if (!child3)
966             return;
967         use(child3);
968     }
969 }
970
971 void SpeculativeJIT::compileGetById(Node* node, AccessType accessType)
972 {
973     ASSERT(accessType == AccessType::Get || accessType == AccessType::GetDirect || accessType == AccessType::TryGet);
974
975     switch (node->child1().useKind()) {
976     case CellUse: {
977         SpeculateCellOperand base(this, node->child1());
978         JSValueRegsTemporary result(this, Reuse, base);
979
980         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
981         JSValueRegs resultRegs = result.regs();
982
983         base.use();
984
985         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, accessType);
986
987         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
988         break;
989     }
990
991     case UntypedUse: {
992         JSValueOperand base(this, node->child1());
993         JSValueRegsTemporary result(this, Reuse, base);
994
995         JSValueRegs baseRegs = base.jsValueRegs();
996         JSValueRegs resultRegs = result.regs();
997
998         base.use();
999
1000         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1001
1002         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, accessType);
1003
1004         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1005         break;
1006     }
1007
1008     default:
1009         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1010         break;
1011     }
1012 }
1013
1014 void SpeculativeJIT::compileGetByIdFlush(Node* node, AccessType accessType)
1015 {
1016     switch (node->child1().useKind()) {
1017     case CellUse: {
1018         SpeculateCellOperand base(this, node->child1());
1019         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
1020
1021         JSValueRegsFlushedCallResult result(this);
1022         JSValueRegs resultRegs = result.regs();
1023
1024         base.use();
1025
1026         flushRegisters();
1027
1028         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), DontSpill, accessType);
1029
1030         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1031         break;
1032     }
1033
1034     case UntypedUse: {
1035         JSValueOperand base(this, node->child1());
1036         JSValueRegs baseRegs = base.jsValueRegs();
1037
1038         JSValueRegsFlushedCallResult result(this);
1039         JSValueRegs resultRegs = result.regs();
1040
1041         base.use();
1042
1043         flushRegisters();
1044
1045         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1046
1047         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, DontSpill, accessType);
1048
1049         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1050         break;
1051     }
1052
1053     default:
1054         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1055         break;
1056     }
1057 }
1058
1059 void SpeculativeJIT::compileInById(Node* node)
1060 {
1061     SpeculateCellOperand base(this, node->child1());
1062     JSValueRegsTemporary result(this, Reuse, base, PayloadWord);
1063
1064     GPRReg baseGPR = base.gpr();
1065     JSValueRegs resultRegs = result.regs();
1066
1067     base.use();
1068
1069     CodeOrigin codeOrigin = node->origin.semantic;
1070     CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size());
1071     RegisterSet usedRegisters = this->usedRegisters();
1072     JITInByIdGenerator gen(
1073         m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, identifierUID(node->identifierNumber()),
1074         JSValueRegs::payloadOnly(baseGPR), resultRegs);
1075     gen.generateFastPath(m_jit);
1076
1077     auto slowPath = slowPathCall(
1078         gen.slowPathJump(), this, operationInByIdOptimize,
1079         NeedToSpill, ExceptionCheckRequirement::CheckNeeded,
1080         resultRegs, gen.stubInfo(), CCallHelpers::CellValue(baseGPR), identifierUID(node->identifierNumber()));
1081
1082     m_jit.addInById(gen, slowPath.get());
1083     addSlowPathGenerator(WTFMove(slowPath));
1084
1085     blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1086 }
1087
1088 void SpeculativeJIT::compileInByVal(Node* node)
1089 {
1090     SpeculateCellOperand base(this, node->child1());
1091     JSValueOperand key(this, node->child2());
1092
1093     GPRReg baseGPR = base.gpr();
1094     JSValueRegs regs = key.jsValueRegs();
1095
1096     base.use();
1097     key.use();
1098
1099     flushRegisters();
1100     JSValueRegsFlushedCallResult result(this);
1101     JSValueRegs resultRegs = result.regs();
1102     callOperation(operationInByVal, resultRegs, baseGPR, regs);
1103     m_jit.exceptionCheck();
1104     blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1105 }
1106
1107 void SpeculativeJIT::compileDeleteById(Node* node)
1108 {
1109     JSValueOperand value(this, node->child1());
1110     GPRFlushedCallResult result(this);
1111
1112     JSValueRegs valueRegs = value.jsValueRegs();
1113     GPRReg resultGPR = result.gpr();
1114
1115     value.use();
1116
1117     flushRegisters();
1118     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1119     m_jit.exceptionCheck();
1120
1121     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1122 }
1123
1124 void SpeculativeJIT::compileDeleteByVal(Node* node)
1125 {
1126     JSValueOperand base(this, node->child1());
1127     JSValueOperand key(this, node->child2());
1128     GPRFlushedCallResult result(this);
1129
1130     JSValueRegs baseRegs = base.jsValueRegs();
1131     JSValueRegs keyRegs = key.jsValueRegs();
1132     GPRReg resultGPR = result.gpr();
1133
1134     base.use();
1135     key.use();
1136
1137     flushRegisters();
1138     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1139     m_jit.exceptionCheck();
1140
1141     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1142 }
1143
1144 void SpeculativeJIT::compilePushWithScope(Node* node)
1145 {
1146     SpeculateCellOperand currentScope(this, node->child1());
1147     GPRReg currentScopeGPR = currentScope.gpr();
1148
1149     GPRFlushedCallResult result(this);
1150     GPRReg resultGPR = result.gpr();
1151
1152     auto objectEdge = node->child2();
1153     if (objectEdge.useKind() == ObjectUse) {
1154         SpeculateCellOperand object(this, objectEdge);
1155         GPRReg objectGPR = object.gpr();
1156         speculateObject(objectEdge, objectGPR);
1157
1158         flushRegisters();
1159         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1160         // No exception check here as we did not have to call toObject().
1161     } else {
1162         ASSERT(objectEdge.useKind() == UntypedUse);
1163         JSValueOperand object(this, objectEdge);
1164         JSValueRegs objectRegs = object.jsValueRegs();
1165
1166         flushRegisters();
1167         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1168         m_jit.exceptionCheck();
1169     }
1170     
1171     cellResult(resultGPR, node);
1172 }
1173
1174 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1175 {
1176     unsigned branchIndexInBlock = detectPeepHoleBranch();
1177     if (branchIndexInBlock != UINT_MAX) {
1178         Node* branchNode = m_block->at(branchIndexInBlock);
1179
1180         ASSERT(node->adjustedRefCount() == 1);
1181         
1182         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1183     
1184         m_indexInBlock = branchIndexInBlock;
1185         m_currentNode = branchNode;
1186         
1187         return true;
1188     }
1189     
1190     nonSpeculativeNonPeepholeStrictEq(node, invert);
1191     
1192     return false;
1193 }
1194
1195 static const char* dataFormatString(DataFormat format)
1196 {
1197     // These values correspond to the DataFormat enum.
1198     const char* strings[] = {
1199         "[  ]",
1200         "[ i]",
1201         "[ d]",
1202         "[ c]",
1203         "Err!",
1204         "Err!",
1205         "Err!",
1206         "Err!",
1207         "[J ]",
1208         "[Ji]",
1209         "[Jd]",
1210         "[Jc]",
1211         "Err!",
1212         "Err!",
1213         "Err!",
1214         "Err!",
1215     };
1216     return strings[format];
1217 }
1218
1219 void SpeculativeJIT::dump(const char* label)
1220 {
1221     if (label)
1222         dataLogF("<%s>\n", label);
1223
1224     dataLogF("  gprs:\n");
1225     m_gprs.dump();
1226     dataLogF("  fprs:\n");
1227     m_fprs.dump();
1228     dataLogF("  VirtualRegisters:\n");
1229     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1230         GenerationInfo& info = m_generationInfo[i];
1231         if (info.alive())
1232             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1233         else
1234             dataLogF("    % 3d:[__][__]", i);
1235         if (info.registerFormat() == DataFormatDouble)
1236             dataLogF(":fpr%d\n", info.fpr());
1237         else if (info.registerFormat() != DataFormatNone
1238 #if USE(JSVALUE32_64)
1239             && !(info.registerFormat() & DataFormatJS)
1240 #endif
1241             ) {
1242             ASSERT(info.gpr() != InvalidGPRReg);
1243             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1244         } else
1245             dataLogF("\n");
1246     }
1247     if (label)
1248         dataLogF("</%s>\n", label);
1249 }
1250
1251 GPRTemporary::GPRTemporary()
1252     : m_jit(0)
1253     , m_gpr(InvalidGPRReg)
1254 {
1255 }
1256
1257 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1258     : m_jit(jit)
1259     , m_gpr(InvalidGPRReg)
1260 {
1261     m_gpr = m_jit->allocate();
1262 }
1263
1264 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1265     : m_jit(jit)
1266     , m_gpr(InvalidGPRReg)
1267 {
1268     m_gpr = m_jit->allocate(specific);
1269 }
1270
1271 #if USE(JSVALUE32_64)
1272 GPRTemporary::GPRTemporary(
1273     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1274     : m_jit(jit)
1275     , m_gpr(InvalidGPRReg)
1276 {
1277     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1278         m_gpr = m_jit->reuse(op1.gpr(which));
1279     else
1280         m_gpr = m_jit->allocate();
1281 }
1282 #else // USE(JSVALUE32_64)
1283 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord)
1284     : GPRTemporary(jit, Reuse, op1)
1285 {
1286 }
1287 #endif
1288
1289 JSValueRegsTemporary::JSValueRegsTemporary() { }
1290
1291 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1292 #if USE(JSVALUE64)
1293     : m_gpr(jit)
1294 #else
1295     : m_payloadGPR(jit)
1296     , m_tagGPR(jit)
1297 #endif
1298 {
1299 }
1300
1301 #if USE(JSVALUE64)
1302 template<typename T>
1303 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1304     : m_gpr(jit, Reuse, operand)
1305 {
1306 }
1307 #else
1308 template<typename T>
1309 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1310 {
1311     if (resultWord == PayloadWord) {
1312         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1313         m_tagGPR = GPRTemporary(jit);
1314     } else {
1315         m_payloadGPR = GPRTemporary(jit);
1316         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1317     }
1318 }
1319 #endif
1320
1321 #if USE(JSVALUE64)
1322 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1323 {
1324     m_gpr = GPRTemporary(jit, Reuse, operand);
1325 }
1326 #else
1327 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1328 {
1329     if (jit->canReuse(operand.node())) {
1330         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1331         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1332     } else {
1333         m_payloadGPR = GPRTemporary(jit);
1334         m_tagGPR = GPRTemporary(jit);
1335     }
1336 }
1337 #endif
1338
1339 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1340
1341 JSValueRegs JSValueRegsTemporary::regs()
1342 {
1343 #if USE(JSVALUE64)
1344     return JSValueRegs(m_gpr.gpr());
1345 #else
1346     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1347 #endif
1348 }
1349
1350 void GPRTemporary::adopt(GPRTemporary& other)
1351 {
1352     ASSERT(!m_jit);
1353     ASSERT(m_gpr == InvalidGPRReg);
1354     ASSERT(other.m_jit);
1355     ASSERT(other.m_gpr != InvalidGPRReg);
1356     m_jit = other.m_jit;
1357     m_gpr = other.m_gpr;
1358     other.m_jit = 0;
1359     other.m_gpr = InvalidGPRReg;
1360 }
1361
1362 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1363 {
1364     ASSERT(other.m_jit);
1365     ASSERT(other.m_fpr != InvalidFPRReg);
1366     m_jit = other.m_jit;
1367     m_fpr = other.m_fpr;
1368
1369     other.m_jit = nullptr;
1370 }
1371
1372 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1373     : m_jit(jit)
1374     , m_fpr(InvalidFPRReg)
1375 {
1376     m_fpr = m_jit->fprAllocate();
1377 }
1378
1379 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1380     : m_jit(jit)
1381     , m_fpr(InvalidFPRReg)
1382 {
1383     if (m_jit->canReuse(op1.node()))
1384         m_fpr = m_jit->reuse(op1.fpr());
1385     else
1386         m_fpr = m_jit->fprAllocate();
1387 }
1388
1389 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1390     : m_jit(jit)
1391     , m_fpr(InvalidFPRReg)
1392 {
1393     if (m_jit->canReuse(op1.node()))
1394         m_fpr = m_jit->reuse(op1.fpr());
1395     else if (m_jit->canReuse(op2.node()))
1396         m_fpr = m_jit->reuse(op2.fpr());
1397     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1398         m_fpr = m_jit->reuse(op1.fpr());
1399     else
1400         m_fpr = m_jit->fprAllocate();
1401 }
1402
1403 #if USE(JSVALUE32_64)
1404 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1405     : m_jit(jit)
1406     , m_fpr(InvalidFPRReg)
1407 {
1408     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1409         m_fpr = m_jit->reuse(op1.fpr());
1410     else
1411         m_fpr = m_jit->fprAllocate();
1412 }
1413 #endif
1414
1415 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1416 {
1417     BasicBlock* taken = branchNode->branchData()->taken.block;
1418     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1419
1420     if (taken == nextBlock()) {
1421         condition = MacroAssembler::invert(condition);
1422         std::swap(taken, notTaken);
1423     }
1424
1425     SpeculateDoubleOperand op1(this, node->child1());
1426     SpeculateDoubleOperand op2(this, node->child2());
1427     
1428     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1429     jump(notTaken);
1430 }
1431
1432 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1433 {
1434     BasicBlock* taken = branchNode->branchData()->taken.block;
1435     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1436
1437     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1438     
1439     if (taken == nextBlock()) {
1440         condition = MacroAssembler::NotEqual;
1441         BasicBlock* tmp = taken;
1442         taken = notTaken;
1443         notTaken = tmp;
1444     }
1445
1446     SpeculateCellOperand op1(this, node->child1());
1447     SpeculateCellOperand op2(this, node->child2());
1448     
1449     GPRReg op1GPR = op1.gpr();
1450     GPRReg op2GPR = op2.gpr();
1451     
1452     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1453         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1454             speculationCheck(
1455                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1456         }
1457         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1458             speculationCheck(
1459                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1460         }
1461     } else {
1462         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1463             speculationCheck(
1464                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1465                 m_jit.branchIfNotObject(op1GPR));
1466         }
1467         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1468             m_jit.branchTest8(
1469                 MacroAssembler::NonZero, 
1470                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1471                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1472
1473         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1474             speculationCheck(
1475                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1476                 m_jit.branchIfNotObject(op2GPR));
1477         }
1478         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1479             m_jit.branchTest8(
1480                 MacroAssembler::NonZero, 
1481                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1482                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1483     }
1484
1485     branchPtr(condition, op1GPR, op2GPR, taken);
1486     jump(notTaken);
1487 }
1488
1489 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1490 {
1491     BasicBlock* taken = branchNode->branchData()->taken.block;
1492     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1493
1494     // The branch instruction will branch to the taken block.
1495     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1496     if (taken == nextBlock()) {
1497         condition = JITCompiler::invert(condition);
1498         BasicBlock* tmp = taken;
1499         taken = notTaken;
1500         notTaken = tmp;
1501     }
1502
1503     if (node->child1()->isInt32Constant()) {
1504         int32_t imm = node->child1()->asInt32();
1505         SpeculateBooleanOperand op2(this, node->child2());
1506         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1507     } else if (node->child2()->isInt32Constant()) {
1508         SpeculateBooleanOperand op1(this, node->child1());
1509         int32_t imm = node->child2()->asInt32();
1510         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1511     } else {
1512         SpeculateBooleanOperand op1(this, node->child1());
1513         SpeculateBooleanOperand op2(this, node->child2());
1514         branch32(condition, op1.gpr(), op2.gpr(), taken);
1515     }
1516
1517     jump(notTaken);
1518 }
1519
1520 void SpeculativeJIT::compileStringSlice(Node* node)
1521 {
1522     SpeculateCellOperand string(this, node->child1());
1523     GPRTemporary startIndex(this);
1524     GPRTemporary temp(this);
1525     GPRTemporary temp2(this);
1526
1527     GPRReg stringGPR = string.gpr();
1528     GPRReg startIndexGPR = startIndex.gpr();
1529     GPRReg tempGPR = temp.gpr();
1530     GPRReg temp2GPR = temp2.gpr();
1531
1532     speculateString(node->child1(), stringGPR);
1533
1534     {
1535         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1536
1537         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1538         if (node->child3())
1539             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1540         else
1541             m_jit.move(temp2GPR, tempGPR);
1542     }
1543
1544     CCallHelpers::JumpList doneCases;
1545     CCallHelpers::JumpList slowCases;
1546
1547     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1548     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1549     doneCases.append(m_jit.jump());
1550
1551     nonEmptyCase.link(&m_jit);
1552     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1553     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1554
1555     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1556     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1557
1558     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1559
1560     // Load the character into scratchReg
1561     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1562     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1563
1564     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1565     auto cont8Bit = m_jit.jump();
1566
1567     is16Bit.link(&m_jit);
1568     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1569
1570     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1571
1572     // 8 bit string values don't need the isASCII check.
1573     cont8Bit.link(&m_jit);
1574
1575     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1576     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1577     m_jit.loadPtr(tempGPR, tempGPR);
1578
1579     addSlowPathGenerator(
1580         slowPathCall(
1581             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1582
1583     addSlowPathGenerator(
1584         slowPathCall(
1585             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1586
1587     doneCases.link(&m_jit);
1588     cellResult(tempGPR, node);
1589 }
1590
1591 void SpeculativeJIT::compileToLowerCase(Node* node)
1592 {
1593     ASSERT(node->op() == ToLowerCase);
1594     SpeculateCellOperand string(this, node->child1());
1595     GPRTemporary temp(this);
1596     GPRTemporary index(this);
1597     GPRTemporary charReg(this);
1598     GPRTemporary length(this);
1599
1600     GPRReg stringGPR = string.gpr();
1601     GPRReg tempGPR = temp.gpr();
1602     GPRReg indexGPR = index.gpr();
1603     GPRReg charGPR = charReg.gpr();
1604     GPRReg lengthGPR = length.gpr();
1605
1606     speculateString(node->child1(), stringGPR);
1607
1608     CCallHelpers::JumpList slowPath;
1609
1610     m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1611
1612     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1613     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1614
1615     slowPath.append(m_jit.branchTest32(
1616         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1617         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1618     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1619     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1620
1621     auto loopStart = m_jit.label();
1622     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1623     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1624     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1625     m_jit.sub32(TrustedImm32('A'), charGPR);
1626     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1627
1628     m_jit.add32(TrustedImm32(1), indexGPR);
1629     m_jit.jump().linkTo(loopStart, &m_jit);
1630     
1631     slowPath.link(&m_jit);
1632     silentSpillAllRegisters(lengthGPR);
1633     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1634     silentFillAllRegisters();
1635     m_jit.exceptionCheck();
1636     auto done = m_jit.jump();
1637
1638     loopDone.link(&m_jit);
1639     m_jit.move(stringGPR, lengthGPR);
1640
1641     done.link(&m_jit);
1642     cellResult(lengthGPR, node);
1643 }
1644
1645 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1646 {
1647     BasicBlock* taken = branchNode->branchData()->taken.block;
1648     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1649
1650     // The branch instruction will branch to the taken block.
1651     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1652     if (taken == nextBlock()) {
1653         condition = JITCompiler::invert(condition);
1654         BasicBlock* tmp = taken;
1655         taken = notTaken;
1656         notTaken = tmp;
1657     }
1658
1659     if (node->child1()->isInt32Constant()) {
1660         int32_t imm = node->child1()->asInt32();
1661         SpeculateInt32Operand op2(this, node->child2());
1662         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1663     } else if (node->child2()->isInt32Constant()) {
1664         SpeculateInt32Operand op1(this, node->child1());
1665         int32_t imm = node->child2()->asInt32();
1666         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1667     } else {
1668         SpeculateInt32Operand op1(this, node->child1());
1669         SpeculateInt32Operand op2(this, node->child2());
1670         branch32(condition, op1.gpr(), op2.gpr(), taken);
1671     }
1672
1673     jump(notTaken);
1674 }
1675
1676 // Returns true if the compare is fused with a subsequent branch.
1677 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1678 {
1679     // Fused compare & branch.
1680     unsigned branchIndexInBlock = detectPeepHoleBranch();
1681     if (branchIndexInBlock != UINT_MAX) {
1682         Node* branchNode = m_block->at(branchIndexInBlock);
1683
1684         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1685         // so can be no intervening nodes to also reference the compare. 
1686         ASSERT(node->adjustedRefCount() == 1);
1687
1688         if (node->isBinaryUseKind(Int32Use))
1689             compilePeepHoleInt32Branch(node, branchNode, condition);
1690 #if USE(JSVALUE64)
1691         else if (node->isBinaryUseKind(Int52RepUse))
1692             compilePeepHoleInt52Branch(node, branchNode, condition);
1693 #endif // USE(JSVALUE64)
1694         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1695             // Use non-peephole comparison, for now.
1696             return false;
1697         } else if (node->isBinaryUseKind(DoubleRepUse))
1698             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1699         else if (node->op() == CompareEq) {
1700             if (node->isBinaryUseKind(BooleanUse))
1701                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1702             else if (node->isBinaryUseKind(SymbolUse))
1703                 compilePeepHoleSymbolEquality(node, branchNode);
1704             else if (node->isBinaryUseKind(ObjectUse))
1705                 compilePeepHoleObjectEquality(node, branchNode);
1706             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1707                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1708             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1709                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1710             else if (!needsTypeCheck(node->child1(), SpecOther))
1711                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1712             else if (!needsTypeCheck(node->child2(), SpecOther))
1713                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1714             else {
1715                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1716                 return true;
1717             }
1718         } else {
1719             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1720             return true;
1721         }
1722
1723         use(node->child1());
1724         use(node->child2());
1725         m_indexInBlock = branchIndexInBlock;
1726         m_currentNode = branchNode;
1727         return true;
1728     }
1729     return false;
1730 }
1731
1732 void SpeculativeJIT::noticeOSRBirth(Node* node)
1733 {
1734     if (!node->hasVirtualRegister())
1735         return;
1736     
1737     VirtualRegister virtualRegister = node->virtualRegister();
1738     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1739     
1740     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1741 }
1742
1743 void SpeculativeJIT::compileMovHint(Node* node)
1744 {
1745     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1746     
1747     Node* child = node->child1().node();
1748     noticeOSRBirth(child);
1749     
1750     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1751 }
1752
1753 void SpeculativeJIT::bail(AbortReason reason)
1754 {
1755     if (verboseCompilationEnabled())
1756         dataLog("Bailing compilation.\n");
1757     m_compileOkay = true;
1758     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1759     clearGenerationInfo();
1760 }
1761
1762 void SpeculativeJIT::compileCurrentBlock()
1763 {
1764     ASSERT(m_compileOkay);
1765     
1766     if (!m_block)
1767         return;
1768     
1769     ASSERT(m_block->isReachable);
1770     
1771     m_jit.blockHeads()[m_block->index] = m_jit.label();
1772
1773     if (!m_block->intersectionOfCFAHasVisited) {
1774         // Don't generate code for basic blocks that are unreachable according to CFA.
1775         // But to be sure that nobody has generated a jump to this block, drop in a
1776         // breakpoint here.
1777         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1778         return;
1779     }
1780
1781     if (m_block->isCatchEntrypoint) {
1782         m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1783         if (Options::zeroStackFrame())
1784             m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1785         m_jit.emitSaveCalleeSaves();
1786         m_jit.emitMaterializeTagCheckRegisters();
1787         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1788     }
1789
1790     m_stream->appendAndLog(VariableEvent::reset());
1791     
1792     m_jit.jitAssertHasValidCallFrame();
1793     m_jit.jitAssertTagsInPlace();
1794     m_jit.jitAssertArgumentCountSane();
1795
1796     m_state.reset();
1797     m_state.beginBasicBlock(m_block);
1798     
1799     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1800         int operand = m_block->variablesAtHead.operandForIndex(i);
1801         Node* node = m_block->variablesAtHead[i];
1802         if (!node)
1803             continue; // No need to record dead SetLocal's.
1804         
1805         VariableAccessData* variable = node->variableAccessData();
1806         DataFormat format;
1807         if (!node->refCount())
1808             continue; // No need to record dead SetLocal's.
1809         format = dataFormatFor(variable->flushFormat());
1810         m_stream->appendAndLog(
1811             VariableEvent::setLocal(
1812                 VirtualRegister(operand),
1813                 variable->machineLocal(),
1814                 format));
1815     }
1816
1817     m_origin = NodeOrigin();
1818     
1819     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1820         m_currentNode = m_block->at(m_indexInBlock);
1821         
1822         // We may have hit a contradiction that the CFA was aware of but that the JIT
1823         // didn't cause directly.
1824         if (!m_state.isValid()) {
1825             bail(DFGBailedAtTopOfBlock);
1826             return;
1827         }
1828
1829         m_interpreter.startExecuting();
1830         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1831         m_jit.setForNode(m_currentNode);
1832         m_origin = m_currentNode->origin;
1833         if (validationEnabled())
1834             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1835         m_lastGeneratedNode = m_currentNode->op();
1836         
1837         ASSERT(m_currentNode->shouldGenerate());
1838         
1839         if (verboseCompilationEnabled()) {
1840             dataLogF(
1841                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1842                 (int)m_currentNode->index(),
1843                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1844             dataLog("\n");
1845         }
1846
1847         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1848             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1849
1850         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1851
1852         compile(m_currentNode);
1853         
1854         if (belongsInMinifiedGraph(m_currentNode->op()))
1855             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1856         
1857 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1858         m_jit.clearRegisterAllocationOffsets();
1859 #endif
1860         
1861         if (!m_compileOkay) {
1862             bail(DFGBailedAtEndOfNode);
1863             return;
1864         }
1865         
1866         // Make sure that the abstract state is rematerialized for the next node.
1867         m_interpreter.executeEffects(m_indexInBlock);
1868     }
1869     
1870     // Perform the most basic verification that children have been used correctly.
1871     if (!ASSERT_DISABLED) {
1872         for (auto& info : m_generationInfo)
1873             RELEASE_ASSERT(!info.alive());
1874     }
1875 }
1876
1877 // If we are making type predictions about our arguments then
1878 // we need to check that they are correct on function entry.
1879 void SpeculativeJIT::checkArgumentTypes()
1880 {
1881     ASSERT(!m_currentNode);
1882     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1883
1884     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1885     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1886         Node* node = arguments[i];
1887         if (!node) {
1888             // The argument is dead. We don't do any checks for such arguments.
1889             continue;
1890         }
1891         
1892         ASSERT(node->op() == SetArgument);
1893         ASSERT(node->shouldGenerate());
1894
1895         VariableAccessData* variableAccessData = node->variableAccessData();
1896         FlushFormat format = variableAccessData->flushFormat();
1897         
1898         if (format == FlushedJSValue)
1899             continue;
1900         
1901         VirtualRegister virtualRegister = variableAccessData->local();
1902
1903         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1904         
1905 #if USE(JSVALUE64)
1906         switch (format) {
1907         case FlushedInt32: {
1908             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1909             break;
1910         }
1911         case FlushedBoolean: {
1912             GPRTemporary temp(this);
1913             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1914             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1915             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1916             break;
1917         }
1918         case FlushedCell: {
1919             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1920             break;
1921         }
1922         default:
1923             RELEASE_ASSERT_NOT_REACHED();
1924             break;
1925         }
1926 #else
1927         switch (format) {
1928         case FlushedInt32: {
1929             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1930             break;
1931         }
1932         case FlushedBoolean: {
1933             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1934             break;
1935         }
1936         case FlushedCell: {
1937             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1938             break;
1939         }
1940         default:
1941             RELEASE_ASSERT_NOT_REACHED();
1942             break;
1943         }
1944 #endif
1945     }
1946
1947     m_origin = NodeOrigin();
1948 }
1949
1950 bool SpeculativeJIT::compile()
1951 {
1952     checkArgumentTypes();
1953     
1954     ASSERT(!m_currentNode);
1955     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1956         m_jit.setForBlockIndex(blockIndex);
1957         m_block = m_jit.graph().block(blockIndex);
1958         compileCurrentBlock();
1959     }
1960     linkBranches();
1961     return true;
1962 }
1963
1964 void SpeculativeJIT::createOSREntries()
1965 {
1966     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1967         BasicBlock* block = m_jit.graph().block(blockIndex);
1968         if (!block)
1969             continue;
1970         if (block->isOSRTarget || block->isCatchEntrypoint) {
1971             // Currently we don't have OSR entry trampolines. We could add them
1972             // here if need be.
1973             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1974         }
1975     }
1976 }
1977
1978 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1979 {
1980     unsigned osrEntryIndex = 0;
1981     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1982         BasicBlock* block = m_jit.graph().block(blockIndex);
1983         if (!block)
1984             continue;
1985         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1986             continue;
1987         if (block->isCatchEntrypoint) {
1988             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1989             Vector<FlushFormat> argumentFormats;
1990             argumentFormats.reserveInitialCapacity(argumentsVector.size());
1991             for (Node* setArgument : argumentsVector) {
1992                 if (setArgument) {
1993                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
1994                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
1995                     argumentFormats.uncheckedAppend(flushFormat);
1996                 } else
1997                     argumentFormats.uncheckedAppend(DeadFlush);
1998             }
1999             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
2000         } else {
2001             ASSERT(block->isOSRTarget);
2002             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
2003         }
2004     }
2005
2006     m_jit.jitCode()->finalizeOSREntrypoints();
2007     m_jit.jitCode()->common.finalizeCatchEntrypoints();
2008
2009     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
2010     
2011     if (verboseCompilationEnabled()) {
2012         DumpContext dumpContext;
2013         dataLog("OSR Entries:\n");
2014         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
2015             dataLog("    ", inContext(entryData, &dumpContext), "\n");
2016         if (!dumpContext.isEmpty())
2017             dumpContext.dump(WTF::dataFile());
2018     }
2019 }
2020     
2021 void SpeculativeJIT::compileCheckTraps(Node*)
2022 {
2023     ASSERT(Options::usePollingTraps());
2024     GPRTemporary unused(this);
2025     GPRReg unusedGPR = unused.gpr();
2026
2027     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2028         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2029
2030     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2031 }
2032
2033 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2034 {
2035     Edge child3 = m_jit.graph().varArgChild(node, 2);
2036     Edge child4 = m_jit.graph().varArgChild(node, 3);
2037
2038     ArrayMode arrayMode = node->arrayMode();
2039     
2040     GPRReg baseReg = base.gpr();
2041     GPRReg propertyReg = property.gpr();
2042     
2043     SpeculateDoubleOperand value(this, child3);
2044
2045     FPRReg valueReg = value.fpr();
2046     
2047     DFG_TYPE_CHECK(
2048         JSValueRegs(), child3, SpecFullRealNumber,
2049         m_jit.branchDouble(
2050             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2051     
2052     if (!m_compileOkay)
2053         return;
2054     
2055     StorageOperand storage(this, child4);
2056     GPRReg storageReg = storage.gpr();
2057
2058     if (node->op() == PutByValAlias) {
2059         // Store the value to the array.
2060         GPRReg propertyReg = property.gpr();
2061         FPRReg valueReg = value.fpr();
2062         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2063         
2064         noResult(m_currentNode);
2065         return;
2066     }
2067     
2068     GPRTemporary temporary;
2069     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2070
2071     MacroAssembler::Jump slowCase;
2072     
2073     if (arrayMode.isInBounds()) {
2074         speculationCheck(
2075             OutOfBounds, JSValueRegs(), 0,
2076             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2077     } else {
2078         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2079         
2080         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2081         
2082         if (!arrayMode.isOutOfBounds())
2083             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2084         
2085         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2086         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2087
2088         inBounds.link(&m_jit);
2089     }
2090     
2091     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2092
2093     base.use();
2094     property.use();
2095     value.use();
2096     storage.use();
2097     
2098     if (arrayMode.isOutOfBounds()) {
2099         addSlowPathGenerator(
2100             slowPathCall(
2101                 slowCase, this,
2102                 m_jit.codeBlock()->isStrictMode()
2103                     ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2104                     : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2105                 NoResult, baseReg, propertyReg, valueReg));
2106     }
2107
2108     noResult(m_currentNode, UseChildrenCalledExplicitly);
2109 }
2110
2111 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2112 {
2113     SpeculateCellOperand string(this, node->child1());
2114     SpeculateStrictInt32Operand index(this, node->child2());
2115     StorageOperand storage(this, node->child3());
2116
2117     GPRReg stringReg = string.gpr();
2118     GPRReg indexReg = index.gpr();
2119     GPRReg storageReg = storage.gpr();
2120     
2121     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2122
2123     // unsigned comparison so we can filter out negative indices and indices that are too large
2124     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2125
2126     GPRTemporary scratch(this);
2127     GPRReg scratchReg = scratch.gpr();
2128
2129     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2130
2131     // Load the character into scratchReg
2132     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2133
2134     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2135     JITCompiler::Jump cont8Bit = m_jit.jump();
2136
2137     is16Bit.link(&m_jit);
2138
2139     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2140
2141     cont8Bit.link(&m_jit);
2142
2143     int32Result(scratchReg, m_currentNode);
2144 }
2145
2146 void SpeculativeJIT::compileGetByValOnString(Node* node)
2147 {
2148     SpeculateCellOperand base(this, m_graph.child(node, 0));
2149     SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2150     StorageOperand storage(this, m_graph.child(node, 2));
2151     GPRReg baseReg = base.gpr();
2152     GPRReg propertyReg = property.gpr();
2153     GPRReg storageReg = storage.gpr();
2154
2155     GPRTemporary scratch(this);
2156     GPRReg scratchReg = scratch.gpr();
2157 #if USE(JSVALUE32_64)
2158     GPRTemporary resultTag;
2159     GPRReg resultTagReg = InvalidGPRReg;
2160     if (node->arrayMode().isOutOfBounds()) {
2161         GPRTemporary realResultTag(this);
2162         resultTag.adopt(realResultTag);
2163         resultTagReg = resultTag.gpr();
2164     }
2165 #endif
2166
2167     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2168
2169     // unsigned comparison so we can filter out negative indices and indices that are too large
2170     JITCompiler::Jump outOfBounds = m_jit.branch32(
2171         MacroAssembler::AboveOrEqual, propertyReg,
2172         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2173     if (node->arrayMode().isInBounds())
2174         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2175
2176     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2177
2178     // Load the character into scratchReg
2179     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2180
2181     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2182     JITCompiler::Jump cont8Bit = m_jit.jump();
2183
2184     is16Bit.link(&m_jit);
2185
2186     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2187
2188     JITCompiler::Jump bigCharacter =
2189         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2190
2191     // 8 bit string values don't need the isASCII check.
2192     cont8Bit.link(&m_jit);
2193
2194     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2195     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2196     m_jit.loadPtr(scratchReg, scratchReg);
2197
2198     addSlowPathGenerator(
2199         slowPathCall(
2200             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2201
2202     if (node->arrayMode().isOutOfBounds()) {
2203 #if USE(JSVALUE32_64)
2204         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2205 #endif
2206
2207         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2208         bool prototypeChainIsSane = false;
2209         if (globalObject->stringPrototypeChainIsSane()) {
2210             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2211             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2212             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2213             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2214             // indexed properties either.
2215             // https://bugs.webkit.org/show_bug.cgi?id=144668
2216             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure());
2217             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure());
2218             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2219         }
2220         if (prototypeChainIsSane) {
2221 #if USE(JSVALUE64)
2222             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2223                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2224 #else
2225             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2226                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2227                 baseReg, propertyReg));
2228 #endif
2229         } else {
2230 #if USE(JSVALUE64)
2231             addSlowPathGenerator(
2232                 slowPathCall(
2233                     outOfBounds, this, operationGetByValStringInt,
2234                     scratchReg, baseReg, propertyReg));
2235 #else
2236             addSlowPathGenerator(
2237                 slowPathCall(
2238                     outOfBounds, this, operationGetByValStringInt,
2239                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2240 #endif
2241         }
2242         
2243 #if USE(JSVALUE64)
2244         jsValueResult(scratchReg, m_currentNode);
2245 #else
2246         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2247 #endif
2248     } else
2249         cellResult(scratchReg, m_currentNode);
2250 }
2251
2252 void SpeculativeJIT::compileFromCharCode(Node* node)
2253 {
2254     Edge& child = node->child1();
2255     if (child.useKind() == UntypedUse) {
2256         JSValueOperand opr(this, child);
2257         JSValueRegs oprRegs = opr.jsValueRegs();
2258
2259         flushRegisters();
2260         JSValueRegsFlushedCallResult result(this);
2261         JSValueRegs resultRegs = result.regs();
2262         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2263         m_jit.exceptionCheck();
2264         
2265         jsValueResult(resultRegs, node);
2266         return;
2267     }
2268
2269     SpeculateStrictInt32Operand property(this, child);
2270     GPRReg propertyReg = property.gpr();
2271     GPRTemporary smallStrings(this);
2272     GPRTemporary scratch(this);
2273     GPRReg scratchReg = scratch.gpr();
2274     GPRReg smallStringsReg = smallStrings.gpr();
2275
2276     JITCompiler::JumpList slowCases;
2277     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2278     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2279     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2280
2281     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2282     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2283     cellResult(scratchReg, m_currentNode);
2284 }
2285
2286 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2287 {
2288     VirtualRegister virtualRegister = node->virtualRegister();
2289     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2290
2291     switch (info.registerFormat()) {
2292     case DataFormatStorage:
2293         RELEASE_ASSERT_NOT_REACHED();
2294
2295     case DataFormatBoolean:
2296     case DataFormatCell:
2297         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2298         return GeneratedOperandTypeUnknown;
2299
2300     case DataFormatNone:
2301     case DataFormatJSCell:
2302     case DataFormatJS:
2303     case DataFormatJSBoolean:
2304     case DataFormatJSDouble:
2305         return GeneratedOperandJSValue;
2306
2307     case DataFormatJSInt32:
2308     case DataFormatInt32:
2309         return GeneratedOperandInteger;
2310
2311     default:
2312         RELEASE_ASSERT_NOT_REACHED();
2313         return GeneratedOperandTypeUnknown;
2314     }
2315 }
2316
2317 void SpeculativeJIT::compileValueToInt32(Node* node)
2318 {
2319     switch (node->child1().useKind()) {
2320 #if USE(JSVALUE64)
2321     case Int52RepUse: {
2322         SpeculateStrictInt52Operand op1(this, node->child1());
2323         GPRTemporary result(this, Reuse, op1);
2324         GPRReg op1GPR = op1.gpr();
2325         GPRReg resultGPR = result.gpr();
2326         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2327         int32Result(resultGPR, node, DataFormatInt32);
2328         return;
2329     }
2330 #endif // USE(JSVALUE64)
2331         
2332     case DoubleRepUse: {
2333         GPRTemporary result(this);
2334         SpeculateDoubleOperand op1(this, node->child1());
2335         FPRReg fpr = op1.fpr();
2336         GPRReg gpr = result.gpr();
2337         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2338         
2339         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2340             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2341         
2342         int32Result(gpr, node);
2343         return;
2344     }
2345     
2346     case NumberUse:
2347     case NotCellUse: {
2348         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2349         case GeneratedOperandInteger: {
2350             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2351             GPRTemporary result(this, Reuse, op1);
2352             m_jit.move(op1.gpr(), result.gpr());
2353             int32Result(result.gpr(), node, op1.format());
2354             return;
2355         }
2356         case GeneratedOperandJSValue: {
2357             GPRTemporary result(this);
2358 #if USE(JSVALUE64)
2359             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2360
2361             GPRReg gpr = op1.gpr();
2362             GPRReg resultGpr = result.gpr();
2363             FPRTemporary tempFpr(this);
2364             FPRReg fpr = tempFpr.fpr();
2365
2366             JITCompiler::Jump isInteger = m_jit.branchIfInt32(gpr);
2367             JITCompiler::JumpList converted;
2368
2369             if (node->child1().useKind() == NumberUse) {
2370                 DFG_TYPE_CHECK(
2371                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2372                     m_jit.branchIfNotNumber(gpr));
2373             } else {
2374                 JITCompiler::Jump isNumber = m_jit.branchIfNumber(gpr);
2375                 
2376                 DFG_TYPE_CHECK(
2377                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2378                 
2379                 // It's not a cell: so true turns into 1 and all else turns into 0.
2380                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2381                 converted.append(m_jit.jump());
2382                 
2383                 isNumber.link(&m_jit);
2384             }
2385
2386             // First, if we get here we have a double encoded as a JSValue
2387             unboxDouble(gpr, resultGpr, fpr);
2388
2389             silentSpillAllRegisters(resultGpr);
2390             callOperation(operationToInt32, resultGpr, fpr);
2391             silentFillAllRegisters();
2392
2393             converted.append(m_jit.jump());
2394
2395             isInteger.link(&m_jit);
2396             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2397
2398             converted.link(&m_jit);
2399 #else
2400             Node* childNode = node->child1().node();
2401             VirtualRegister virtualRegister = childNode->virtualRegister();
2402             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2403
2404             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2405
2406             GPRReg payloadGPR = op1.payloadGPR();
2407             GPRReg resultGpr = result.gpr();
2408         
2409             JITCompiler::JumpList converted;
2410
2411             if (info.registerFormat() == DataFormatJSInt32)
2412                 m_jit.move(payloadGPR, resultGpr);
2413             else {
2414                 GPRReg tagGPR = op1.tagGPR();
2415                 FPRTemporary tempFpr(this);
2416                 FPRReg fpr = tempFpr.fpr();
2417                 FPRTemporary scratch(this);
2418
2419                 JITCompiler::Jump isInteger = m_jit.branchIfInt32(tagGPR);
2420
2421                 if (node->child1().useKind() == NumberUse) {
2422                     DFG_TYPE_CHECK(
2423                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2424                         m_jit.branch32(
2425                             MacroAssembler::AboveOrEqual, tagGPR,
2426                             TrustedImm32(JSValue::LowestTag)));
2427                 } else {
2428                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2429                     
2430                     DFG_TYPE_CHECK(
2431                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2432                         m_jit.branchIfCell(op1.jsValueRegs()));
2433                     
2434                     // It's not a cell: so true turns into 1 and all else turns into 0.
2435                     JITCompiler::Jump isBoolean = m_jit.branchIfBoolean(tagGPR, InvalidGPRReg);
2436                     m_jit.move(TrustedImm32(0), resultGpr);
2437                     converted.append(m_jit.jump());
2438                     
2439                     isBoolean.link(&m_jit);
2440                     m_jit.move(payloadGPR, resultGpr);
2441                     converted.append(m_jit.jump());
2442                     
2443                     isNumber.link(&m_jit);
2444                 }
2445
2446                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2447
2448                 silentSpillAllRegisters(resultGpr);
2449                 callOperation(operationToInt32, resultGpr, fpr);
2450                 silentFillAllRegisters();
2451
2452                 converted.append(m_jit.jump());
2453
2454                 isInteger.link(&m_jit);
2455                 m_jit.move(payloadGPR, resultGpr);
2456
2457                 converted.link(&m_jit);
2458             }
2459 #endif
2460             int32Result(resultGpr, node);
2461             return;
2462         }
2463         case GeneratedOperandTypeUnknown:
2464             RELEASE_ASSERT(!m_compileOkay);
2465             return;
2466         }
2467         RELEASE_ASSERT_NOT_REACHED();
2468         return;
2469     }
2470     
2471     default:
2472         ASSERT(!m_compileOkay);
2473         return;
2474     }
2475 }
2476
2477 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2478 {
2479     if (doesOverflow(node->arithMode())) {
2480         if (enableInt52()) {
2481             SpeculateInt32Operand op1(this, node->child1());
2482             GPRTemporary result(this, Reuse, op1);
2483             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2484             strictInt52Result(result.gpr(), node);
2485             return;
2486         }
2487         SpeculateInt32Operand op1(this, node->child1());
2488         FPRTemporary result(this);
2489             
2490         GPRReg inputGPR = op1.gpr();
2491         FPRReg outputFPR = result.fpr();
2492             
2493         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2494             
2495         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2496         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2497         positive.link(&m_jit);
2498             
2499         doubleResult(outputFPR, node);
2500         return;
2501     }
2502     
2503     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2504
2505     SpeculateInt32Operand op1(this, node->child1());
2506     GPRTemporary result(this);
2507
2508     m_jit.move(op1.gpr(), result.gpr());
2509
2510     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2511
2512     int32Result(result.gpr(), node, op1.format());
2513 }
2514
2515 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2516 {
2517     SpeculateDoubleOperand op1(this, node->child1());
2518     FPRTemporary scratch(this);
2519     GPRTemporary result(this);
2520     
2521     FPRReg valueFPR = op1.fpr();
2522     FPRReg scratchFPR = scratch.fpr();
2523     GPRReg resultGPR = result.gpr();
2524
2525     JITCompiler::JumpList failureCases;
2526     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2527     m_jit.branchConvertDoubleToInt32(
2528         valueFPR, resultGPR, failureCases, scratchFPR,
2529         shouldCheckNegativeZero(node->arithMode()));
2530     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2531
2532     int32Result(resultGPR, node);
2533 }
2534
2535 void SpeculativeJIT::compileDoubleRep(Node* node)
2536 {
2537     switch (node->child1().useKind()) {
2538     case RealNumberUse: {
2539         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2540         FPRTemporary result(this);
2541         
2542         JSValueRegs op1Regs = op1.jsValueRegs();
2543         FPRReg resultFPR = result.fpr();
2544         
2545 #if USE(JSVALUE64)
2546         GPRTemporary temp(this);
2547         GPRReg tempGPR = temp.gpr();
2548         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2549 #else
2550         FPRTemporary temp(this);
2551         FPRReg tempFPR = temp.fpr();
2552         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2553 #endif
2554         
2555         JITCompiler::Jump done = m_jit.branchDouble(
2556             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2557         
2558         DFG_TYPE_CHECK(
2559             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2560         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2561         
2562         done.link(&m_jit);
2563         
2564         doubleResult(resultFPR, node);
2565         return;
2566     }
2567     
2568     case NotCellUse:
2569     case NumberUse: {
2570         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2571
2572         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2573         if (isInt32Speculation(possibleTypes)) {
2574             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2575             FPRTemporary result(this);
2576             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2577             doubleResult(result.fpr(), node);
2578             return;
2579         }
2580
2581         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2582         FPRTemporary result(this);
2583
2584 #if USE(JSVALUE64)
2585         GPRTemporary temp(this);
2586
2587         GPRReg op1GPR = op1.gpr();
2588         GPRReg tempGPR = temp.gpr();
2589         FPRReg resultFPR = result.fpr();
2590         JITCompiler::JumpList done;
2591
2592         JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1GPR);
2593
2594         if (node->child1().useKind() == NotCellUse) {
2595             JITCompiler::Jump isNumber = m_jit.branchIfNumber(op1GPR);
2596             JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1GPR);
2597
2598             static const double zero = 0;
2599             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2600
2601             JITCompiler::Jump isNull = m_jit.branchIfNull(op1GPR);
2602             done.append(isNull);
2603
2604             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2605                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2606
2607             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2608             static const double one = 1;
2609             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2610             done.append(m_jit.jump());
2611             done.append(isFalse);
2612
2613             isUndefined.link(&m_jit);
2614             static const double NaN = PNaN;
2615             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2616             done.append(m_jit.jump());
2617
2618             isNumber.link(&m_jit);
2619         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2620             typeCheck(
2621                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2622                 m_jit.branchIfNotNumber(op1GPR));
2623         }
2624
2625         unboxDouble(op1GPR, tempGPR, resultFPR);
2626         done.append(m_jit.jump());
2627     
2628         isInteger.link(&m_jit);
2629         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2630         done.link(&m_jit);
2631 #else // USE(JSVALUE64) -> this is the 32_64 case
2632         FPRTemporary temp(this);
2633     
2634         GPRReg op1TagGPR = op1.tagGPR();
2635         GPRReg op1PayloadGPR = op1.payloadGPR();
2636         FPRReg tempFPR = temp.fpr();
2637         FPRReg resultFPR = result.fpr();
2638         JITCompiler::JumpList done;
2639     
2640         JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1TagGPR);
2641
2642         if (node->child1().useKind() == NotCellUse) {
2643             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2644             JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1TagGPR);
2645
2646             static const double zero = 0;
2647             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2648
2649             JITCompiler::Jump isNull = m_jit.branchIfNull(op1TagGPR);
2650             done.append(isNull);
2651
2652             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branchIfNotBoolean(op1TagGPR, InvalidGPRReg));
2653
2654             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2655             static const double one = 1;
2656             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2657             done.append(m_jit.jump());
2658             done.append(isFalse);
2659
2660             isUndefined.link(&m_jit);
2661             static const double NaN = PNaN;
2662             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2663             done.append(m_jit.jump());
2664
2665             isNumber.link(&m_jit);
2666         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2667             // This check fails with Int32Tag, but it is OK since Int32 case is already excluded.
2668             typeCheck(
2669                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2670                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2671         }
2672
2673         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2674         done.append(m_jit.jump());
2675     
2676         isInteger.link(&m_jit);
2677         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2678         done.link(&m_jit);
2679 #endif // USE(JSVALUE64)
2680     
2681         doubleResult(resultFPR, node);
2682         return;
2683     }
2684         
2685 #if USE(JSVALUE64)
2686     case Int52RepUse: {
2687         SpeculateStrictInt52Operand value(this, node->child1());
2688         FPRTemporary result(this);
2689         
2690         GPRReg valueGPR = value.gpr();
2691         FPRReg resultFPR = result.fpr();
2692
2693         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2694         
2695         doubleResult(resultFPR, node);
2696         return;
2697     }
2698 #endif // USE(JSVALUE64)
2699         
2700     default:
2701         RELEASE_ASSERT_NOT_REACHED();
2702         return;
2703     }
2704 }
2705
2706 void SpeculativeJIT::compileValueRep(Node* node)
2707 {
2708     switch (node->child1().useKind()) {
2709     case DoubleRepUse: {
2710         SpeculateDoubleOperand value(this, node->child1());
2711         JSValueRegsTemporary result(this);
2712         
2713         FPRReg valueFPR = value.fpr();
2714         JSValueRegs resultRegs = result.regs();
2715         
2716         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2717         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2718         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2719         // local was purified.
2720         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2721             m_jit.purifyNaN(valueFPR);
2722
2723         boxDouble(valueFPR, resultRegs);
2724         
2725         jsValueResult(resultRegs, node);
2726         return;
2727     }
2728         
2729 #if USE(JSVALUE64)
2730     case Int52RepUse: {
2731         SpeculateStrictInt52Operand value(this, node->child1());
2732         GPRTemporary result(this);
2733         
2734         GPRReg valueGPR = value.gpr();
2735         GPRReg resultGPR = result.gpr();
2736         
2737         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2738         
2739         jsValueResult(resultGPR, node);
2740         return;
2741     }
2742 #endif // USE(JSVALUE64)
2743         
2744     default:
2745         RELEASE_ASSERT_NOT_REACHED();
2746         return;
2747     }
2748 }
2749
2750 static double clampDoubleToByte(double d)
2751 {
2752     d += 0.5;
2753     if (!(d > 0))
2754         d = 0;
2755     else if (d > 255)
2756         d = 255;
2757     return d;
2758 }
2759
2760 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2761 {
2762     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2763     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2764     jit.xorPtr(result, result);
2765     MacroAssembler::Jump clamped = jit.jump();
2766     tooBig.link(&jit);
2767     jit.move(JITCompiler::TrustedImm32(255), result);
2768     clamped.link(&jit);
2769     inBounds.link(&jit);
2770 }
2771
2772 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2773 {
2774     // Unordered compare so we pick up NaN
2775     static const double zero = 0;
2776     static const double byteMax = 255;
2777     static const double half = 0.5;
2778     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2779     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2780     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2781     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2782     
2783     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2784     // FIXME: This should probably just use a floating point round!
2785     // https://bugs.webkit.org/show_bug.cgi?id=72054
2786     jit.addDouble(source, scratch);
2787     jit.truncateDoubleToInt32(scratch, result);   
2788     MacroAssembler::Jump truncatedInt = jit.jump();
2789     
2790     tooSmall.link(&jit);
2791     jit.xorPtr(result, result);
2792     MacroAssembler::Jump zeroed = jit.jump();
2793     
2794     tooBig.link(&jit);
2795     jit.move(JITCompiler::TrustedImm32(255), result);
2796     
2797     truncatedInt.link(&jit);
2798     zeroed.link(&jit);
2799
2800 }
2801
2802 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2803 {
2804     if (node->op() == PutByValAlias)
2805         return JITCompiler::Jump();
2806     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2807         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2808     if (view) {
2809         uint32_t length = view->length();
2810         Node* indexNode = m_jit.graph().child(node, 1).node();
2811         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2812             return JITCompiler::Jump();
2813         return m_jit.branch32(
2814             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2815     }
2816     return m_jit.branch32(
2817         MacroAssembler::AboveOrEqual, indexGPR,
2818         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2819 }
2820
2821 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2822 {
2823     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2824     if (!jump.isSet())
2825         return;
2826     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2827 }
2828
2829 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2830 {
2831     JITCompiler::Jump done;
2832     if (outOfBounds.isSet()) {
2833         done = m_jit.jump();
2834         if (node->arrayMode().isInBounds())
2835             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2836         else {
2837             outOfBounds.link(&m_jit);
2838
2839             JITCompiler::Jump notWasteful = m_jit.branch32(
2840                 MacroAssembler::NotEqual,
2841                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2842                 TrustedImm32(WastefulTypedArray));
2843
2844             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2845                 MacroAssembler::Zero,
2846                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2847             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2848             notWasteful.link(&m_jit);
2849         }
2850     }
2851     return done;
2852 }
2853
2854 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2855 {
2856     switch (elementSize(type)) {
2857     case 1:
2858         if (isSigned(type))
2859             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2860         else
2861             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2862         break;
2863     case 2:
2864         if (isSigned(type))
2865             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2866         else
2867             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2868         break;
2869     case 4:
2870         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2871         break;
2872     default:
2873         CRASH();
2874     }
2875 }
2876
2877 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2878 {
2879     if (elementSize(type) < 4 || isSigned(type)) {
2880         int32Result(resultReg, node);
2881         return;
2882     }
2883     
2884     ASSERT(elementSize(type) == 4 && !isSigned(type));
2885     if (node->shouldSpeculateInt32() && canSpeculate) {
2886         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2887         int32Result(resultReg, node);
2888         return;
2889     }
2890     
2891 #if USE(JSVALUE64)
2892     if (node->shouldSpeculateAnyInt()) {
2893         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2894         strictInt52Result(resultReg, node);
2895         return;
2896     }
2897 #endif
2898     
2899     FPRTemporary fresult(this);
2900     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2901     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2902     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2903     positive.link(&m_jit);
2904     doubleResult(fresult.fpr(), node);
2905 }
2906
2907 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2908 {
2909     ASSERT(isInt(type));
2910     
2911     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2912     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2913     StorageOperand storage(this, m_graph.varArgChild(node, 2));
2914
2915     GPRReg baseReg = base.gpr();
2916     GPRReg propertyReg = property.gpr();
2917     GPRReg storageReg = storage.gpr();
2918
2919     GPRTemporary result(this);
2920     GPRReg resultReg = result.gpr();
2921
2922     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2923
2924     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2925     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2926     bool canSpeculate = true;
2927     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2928 }
2929
2930 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2931     GPRTemporary& value,
2932     GPRReg property,
2933 #if USE(JSVALUE32_64)
2934     GPRTemporary& propertyTag,
2935     GPRTemporary& valueTag,
2936 #endif
2937     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2938 {
2939     bool isAppropriateConstant = false;
2940     if (valueUse->isConstant()) {
2941         JSValue jsValue = valueUse->asJSValue();
2942         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2943         SpeculatedType actualType = speculationFromValue(jsValue);
2944         isAppropriateConstant = (expectedType | actualType) == expectedType;
2945     }
2946     
2947     if (isAppropriateConstant) {
2948         JSValue jsValue = valueUse->asJSValue();
2949         if (!jsValue.isNumber()) {
2950             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2951             return false;
2952         }
2953         double d = jsValue.asNumber();
2954         if (isClamped)
2955             d = clampDoubleToByte(d);
2956         GPRTemporary scratch(this);
2957         GPRReg scratchReg = scratch.gpr();
2958         m_jit.move(Imm32(toInt32(d)), scratchReg);
2959         value.adopt(scratch);
2960     } else {
2961         switch (valueUse.useKind()) {
2962         case Int32Use: {
2963             SpeculateInt32Operand valueOp(this, valueUse);
2964             GPRTemporary scratch(this);
2965             GPRReg scratchReg = scratch.gpr();
2966             m_jit.move(valueOp.gpr(), scratchReg);
2967             if (isClamped)
2968                 compileClampIntegerToByte(m_jit, scratchReg);
2969             value.adopt(scratch);
2970             break;
2971         }
2972             
2973 #if USE(JSVALUE64)
2974         case Int52RepUse: {
2975             SpeculateStrictInt52Operand valueOp(this, valueUse);
2976             GPRTemporary scratch(this);
2977             GPRReg scratchReg = scratch.gpr();
2978             m_jit.move(valueOp.gpr(), scratchReg);
2979             if (isClamped) {
2980                 MacroAssembler::Jump inBounds = m_jit.branch64(
2981                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2982                 MacroAssembler::Jump tooBig = m_jit.branch64(
2983                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2984                 m_jit.move(TrustedImm32(0), scratchReg);
2985                 MacroAssembler::Jump clamped = m_jit.jump();
2986                 tooBig.link(&m_jit);
2987                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2988                 clamped.link(&m_jit);
2989                 inBounds.link(&m_jit);
2990             }
2991             value.adopt(scratch);
2992             break;
2993         }
2994 #endif // USE(JSVALUE64)
2995             
2996         case DoubleRepUse: {
2997             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
2998             if (isClamped) {
2999                 SpeculateDoubleOperand valueOp(this, valueUse);
3000                 GPRTemporary result(this);
3001                 FPRTemporary floatScratch(this);
3002                 FPRReg fpr = valueOp.fpr();
3003                 GPRReg gpr = result.gpr();
3004                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
3005                 value.adopt(result);
3006             } else {
3007 #if USE(JSVALUE32_64)
3008                 GPRTemporary realPropertyTag(this);
3009                 propertyTag.adopt(realPropertyTag);
3010                 GPRReg propertyTagGPR = propertyTag.gpr();
3011
3012                 GPRTemporary realValueTag(this);
3013                 valueTag.adopt(realValueTag);
3014                 GPRReg valueTagGPR = valueTag.gpr();
3015 #endif
3016                 SpeculateDoubleOperand valueOp(this, valueUse);
3017                 GPRTemporary result(this);
3018                 FPRReg fpr = valueOp.fpr();
3019                 GPRReg gpr = result.gpr();
3020                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3021                 m_jit.xorPtr(gpr, gpr);
3022                 MacroAssembler::JumpList fixed(m_jit.jump());
3023                 notNaN.link(&m_jit);
3024
3025                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3026                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3027
3028 #if USE(JSVALUE64)
3029                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3030                 boxDouble(fpr, gpr);
3031 #else
3032                 UNUSED_PARAM(property);
3033                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3034                 boxDouble(fpr, valueTagGPR, gpr);
3035 #endif
3036                 slowPathCases.append(m_jit.jump());
3037
3038                 fixed.link(&m_jit);
3039                 value.adopt(result);
3040             }
3041             break;
3042         }
3043             
3044         default:
3045             RELEASE_ASSERT_NOT_REACHED();
3046             break;
3047         }
3048     }
3049     return true;
3050 }
3051
3052 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3053 {
3054     ASSERT(isInt(type));
3055     
3056     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3057     GPRReg storageReg = storage.gpr();
3058     
3059     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3060     
3061     GPRTemporary value;
3062 #if USE(JSVALUE32_64)
3063     GPRTemporary propertyTag;
3064     GPRTemporary valueTag;
3065 #endif
3066
3067     JITCompiler::JumpList slowPathCases;
3068     
3069     bool result = getIntTypedArrayStoreOperand(
3070         value, property,
3071 #if USE(JSVALUE32_64)
3072         propertyTag, valueTag,
3073 #endif
3074         valueUse, slowPathCases, isClamped(type));
3075     if (!result) {
3076         noResult(node);
3077         return;
3078     }
3079
3080     GPRReg valueGPR = value.gpr();
3081 #if USE(JSVALUE32_64)
3082     GPRReg propertyTagGPR = propertyTag.gpr();
3083     GPRReg valueTagGPR = valueTag.gpr();
3084 #endif
3085
3086     ASSERT_UNUSED(valueGPR, valueGPR != property);
3087     ASSERT(valueGPR != base);
3088     ASSERT(valueGPR != storageReg);
3089     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3090
3091     switch (elementSize(type)) {
3092     case 1:
3093         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3094         break;
3095     case 2:
3096         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3097         break;
3098     case 4:
3099         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3100         break;
3101     default:
3102         CRASH();
3103     }
3104
3105     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3106     if (done.isSet())
3107         done.link(&m_jit);
3108
3109     if (!slowPathCases.empty()) {
3110 #if USE(JSVALUE64)
3111         if (node->op() == PutByValDirect) {
3112             addSlowPathGenerator(slowPathCall(
3113                 slowPathCases, this,
3114                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3115                 NoResult, base, property, valueGPR));
3116         } else {
3117             addSlowPathGenerator(slowPathCall(
3118                 slowPathCases, this,
3119                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3120                 NoResult, base, property, valueGPR));
3121         }
3122 #else // not USE(JSVALUE64)
3123         if (node->op() == PutByValDirect) {
3124             addSlowPathGenerator(slowPathCall(
3125                 slowPathCases, this,
3126                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3127                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3128         } else {
3129             addSlowPathGenerator(slowPathCall(
3130                 slowPathCases, this,
3131                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3132                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3133         }
3134 #endif
3135     }
3136     
3137     noResult(node);
3138 }
3139
3140 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3141 {
3142     ASSERT(isFloat(type));
3143     
3144     SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3145     SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3146     StorageOperand storage(this, m_graph.varArgChild(node, 2));
3147
3148     GPRReg baseReg = base.gpr();
3149     GPRReg propertyReg = property.gpr();
3150     GPRReg storageReg = storage.gpr();
3151
3152     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3153
3154     FPRTemporary result(this);
3155     FPRReg resultReg = result.fpr();
3156     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3157     switch (elementSize(type)) {
3158     case 4:
3159         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3160         m_jit.convertFloatToDouble(resultReg, resultReg);
3161         break;
3162     case 8: {
3163         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3164         break;
3165     }
3166     default:
3167         RELEASE_ASSERT_NOT_REACHED();
3168     }
3169     
3170     doubleResult(resultReg, node);
3171 }
3172
3173 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3174 {
3175     ASSERT(isFloat(type));
3176     
3177     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3178     GPRReg storageReg = storage.gpr();
3179     
3180     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3181     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3182
3183     SpeculateDoubleOperand valueOp(this, valueUse);
3184     FPRTemporary scratch(this);
3185     FPRReg valueFPR = valueOp.fpr();
3186     FPRReg scratchFPR = scratch.fpr();
3187
3188     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3189     
3190     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3191     
3192     switch (elementSize(type)) {
3193     case 4: {
3194         m_jit.moveDouble(valueFPR, scratchFPR);
3195         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3196         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3197         break;
3198     }
3199     case 8:
3200         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3201         break;
3202     default:
3203         RELEASE_ASSERT_NOT_REACHED();
3204     }
3205
3206     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3207     if (done.isSet())
3208         done.link(&m_jit);
3209     noResult(node);
3210 }
3211
3212 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3213 {
3214     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3215     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3216
3217     GPRReg arg1GPR = arg1.gpr();
3218     GPRReg arg2GPR = arg2.gpr();
3219
3220     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3221     speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3222
3223     flushRegisters();
3224     JSValueRegsFlushedCallResult result(this);
3225     JSValueRegs resultRegs = result.regs();
3226     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3227     m_jit.exceptionCheck();
3228
3229     jsValueResult(resultRegs, node);
3230 }
3231
3232 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3233 {
3234     SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3235     SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3236
3237     GPRReg arg1GPR = arg1.gpr();
3238     GPRReg arg2GPR = arg2.gpr();
3239
3240     speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3241     speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3242
3243     flushRegisters();
3244     JSValueRegsFlushedCallResult result(this);
3245     JSValueRegs resultRegs = result.regs();
3246     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3247     m_jit.exceptionCheck();
3248
3249     jsValueResult(resultRegs, node);
3250 }
3251
3252 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3253 {
3254     SpeculateCellOperand arg1(this, child1);
3255     SpeculateCellOperand arg2(this, child2);
3256     JSValueOperand arg3(this, child3);
3257
3258     GPRReg arg1GPR = arg1.gpr();
3259     GPRReg arg2GPR = arg2.gpr();
3260     JSValueRegs arg3Regs = arg3.jsValueRegs();
3261
3262     speculateString(child2, arg2GPR);
3263
3264     flushRegisters();
3265     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3266     m_jit.exceptionCheck();
3267
3268     noResult(node);
3269 }
3270
3271 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3272 {
3273     SpeculateCellOperand arg1(this, child1);
3274     SpeculateCellOperand arg2(this, child2);
3275     JSValueOperand arg3(this, child3);
3276
3277     GPRReg arg1GPR = arg1.gpr();
3278     GPRReg arg2GPR = arg2.gpr();
3279     JSValueRegs arg3Regs = arg3.jsValueRegs();
3280
3281     speculateSymbol(child2, arg2GPR);
3282
3283     flushRegisters();
3284     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3285     m_jit.exceptionCheck();
3286
3287     noResult(node);
3288 }
3289
3290 void SpeculativeJIT::compileGetByValWithThis(Node* node)
3291 {
3292     JSValueOperand base(this, node->child1());
3293     JSValueRegs baseRegs = base.jsValueRegs();
3294     JSValueOperand thisValue(this, node->child2());
3295     JSValueRegs thisValueRegs = thisValue.jsValueRegs();
3296     JSValueOperand subscript(this, node->child3());
3297     JSValueRegs subscriptRegs = subscript.jsValueRegs();
3298
3299     flushRegisters();
3300     JSValueRegsFlushedCallResult result(this);
3301     JSValueRegs resultRegs = result.regs();
3302     callOperation(operationGetByValWithThis, resultRegs, baseRegs, thisValueRegs, subscriptRegs);
3303     m_jit.exceptionCheck();
3304
3305     jsValueResult(resultRegs, node);
3306 }
3307
3308 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3309 {
3310     SpeculateCellOperand base(this, node->child1());
3311
3312     GPRReg baseGPR = base.gpr();
3313
3314     // FIXME: This only works for checking if a single bit is set. If we want to check more
3315     // than one bit at once, we'll need to fix this:
3316     // https://bugs.webkit.org/show_bug.cgi?id=185705
3317     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3318
3319     noResult(node);
3320 }
3321
3322 void SpeculativeJIT::compileParseInt(Node* node)
3323 {
3324     RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3325     if (node->child2()) {
3326         SpeculateInt32Operand radix(this, node->child2());
3327         GPRReg radixGPR = radix.gpr();
3328         if (node->child1().useKind() == UntypedUse) {
3329             JSValueOperand value(this, node->child1());
3330             JSValueRegs valueRegs = value.jsValueRegs();
3331
3332             flushRegisters();
3333             JSValueRegsFlushedCallResult result(this);
3334             JSValueRegs resultRegs = result.regs();
3335             callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3336             m_jit.exceptionCheck();
3337             jsValueResult(resultRegs, node);
3338             return;
3339         }
3340
3341         SpeculateCellOperand value(this, node->child1());
3342         GPRReg valueGPR = value.gpr();
3343         speculateString(node->child1(), valueGPR);
3344
3345         flushRegisters();
3346         JSValueRegsFlushedCallResult result(this);
3347         JSValueRegs resultRegs = result.regs();
3348         callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3349         m_jit.exceptionCheck();
3350         jsValueResult(resultRegs, node);
3351         return;
3352     }
3353
3354     if (node->child1().useKind() == UntypedUse) {
3355         JSValueOperand value(this, node->child1());
3356         JSValueRegs valueRegs = value.jsValueRegs();
3357
3358         flushRegisters();
3359         JSValueRegsFlushedCallResult result(this);
3360         JSValueRegs resultRegs = result.regs();
3361         callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3362         m_jit.exceptionCheck();
3363         jsValueResult(resultRegs, node);
3364         return;
3365     }
3366
3367     SpeculateCellOperand value(this, node->child1());
3368     GPRReg valueGPR = value.gpr();
3369     speculateString(node->child1(), valueGPR);
3370
3371     flushRegisters();
3372     JSValueRegsFlushedCallResult result(this);
3373     JSValueRegs resultRegs = result.regs();
3374     callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3375     m_jit.exceptionCheck();
3376     jsValueResult(resultRegs, node);
3377 }
3378
3379 void SpeculativeJIT::compileOverridesHasInstance(Node* node)
3380 {
3381     Node* hasInstanceValueNode = node->child2().node();
3382     JSFunction* defaultHasInstanceFunction = jsCast<JSFunction*>(node->cellOperand()->value());
3383
3384     MacroAssembler::JumpList notDefault;
3385     SpeculateCellOperand base(this, node->child1());
3386     JSValueOperand hasInstanceValue(this, node->child2());
3387     GPRTemporary result(this);
3388
3389     GPRReg baseGPR = base.gpr();
3390     GPRReg resultGPR = result.gpr();
3391
3392     // It would be great if constant folding handled automatically the case where we knew the hasInstance function
3393     // was a constant. Unfortunately, the folding rule for OverridesHasInstance is in the strength reduction phase
3394     // since it relies on OSR information. https://bugs.webkit.org/show_bug.cgi?id=154832
3395     if (!hasInstanceValueNode->isCellConstant() || defaultHasInstanceFunction != hasInstanceValueNode->asCell()) {
3396         JSValueRegs hasInstanceValueRegs = hasInstanceValue.jsValueRegs();
3397 #if USE(JSVALUE64)
3398         notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.gpr(), TrustedImmPtr(node->cellOperand())));
3399 #else
3400         notDefault.append(m_jit.branchIfNotCell(hasInstanceValueRegs));
3401         notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.payloadGPR(), TrustedImmPtr(node->cellOperand())));
3402 #endif
3403     }
3404
3405     // Check that base 'ImplementsDefaultHasInstance'.
3406     m_jit.test8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance), resultGPR);
3407     MacroAssembler::Jump done = m_jit.jump();
3408
3409     if (!notDefault.empty()) {
3410         notDefault.link(&m_jit);
3411         m_jit.move(TrustedImm32(1), resultGPR);
3412     }
3413
3414     done.link(&m_jit);
3415     unblessedBooleanResult(resultGPR, node);
3416 }
3417
3418 void SpeculativeJIT::compileInstanceOfForCells(Node* node, JSValueRegs valueRegs, JSValueRegs prototypeRegs, GPRReg resultGPR, GPRReg scratchGPR, GPRReg scratch2GPR, JITCompiler::Jump slowCase)
3419 {
3420     CallSiteIndex callSiteIndex = m_jit.addCallSite(node->origin.semantic);
3421     
3422     JITInstanceOfGenerator gen(
3423         m_jit.codeBlock(), node->origin.semantic, callSiteIndex, usedRegisters(), resultGPR,
3424         valueRegs.payloadGPR(), prototypeRegs.payloadGPR(), scratchGPR, scratch2GPR,
3425         m_state.forNode(node->child2()).isType(SpecObject | ~SpecCell));
3426     gen.generateFastPath(m_jit);
3427     
3428     JITCompiler::JumpList slowCases;
3429     slowCases.append(slowCase);
3430     
3431     std::unique_ptr<SlowPathGenerator> slowPath = slowPathCall(
3432         slowCases, this, operationInstanceOfOptimize, resultGPR, gen.stubInfo(), valueRegs,
3433         prototypeRegs);
3434     
3435     m_jit.addInstanceOf(gen, slowPath.get());
3436     addSlowPathGenerator(WTFMove(slowPath));
3437 }
3438
3439 void SpeculativeJIT::compileInstanceOf(Node* node)
3440 {
3441 #if USE(JSVALUE64)
3442     if (node->child1().useKind() == CellUse
3443         && node->child2().useKind() == CellUse) {
3444         SpeculateCellOperand value(this, node->child1());
3445         SpeculateCellOperand prototype(this, node->child2());
3446         
3447         GPRTemporary result(this);
3448         GPRTemporary scratch(this);
3449         GPRTemporary scratch2(this);
3450         
3451         GPRReg valueGPR = value.gpr();
3452         GPRReg prototypeGPR = prototype.gpr();
3453         GPRReg resultGPR = result.gpr();
3454         GPRReg scratchGPR = scratch.gpr();
3455         GPRReg scratch2GPR = scratch2.gpr();
3456         
3457         compileInstanceOfForCells(node, JSValueRegs(valueGPR), JSValueRegs(prototypeGPR), resultGPR, scratchGPR, scratch2GPR);
3458         
3459         blessedBooleanResult(resultGPR, node);
3460         return;
3461     }
3462 #endif
3463     
3464     DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse);
3465     DFG_ASSERT(m_jit.graph(), node, node->child2().useKind() == UntypedUse);
3466     
3467     JSValueOperand value(this, node->child1());
3468     JSValueOperand prototype(this, node->child2());
3469     
3470     GPRTemporary result(this);
3471     GPRTemporary scratch(this);
3472     
3473     JSValueRegs valueRegs = value.jsValueRegs();
3474     JSValueRegs prototypeRegs = prototype.jsValueRegs();
3475     
3476     GPRReg resultGPR = result.gpr();
3477     GPRReg scratchGPR = scratch.gpr();
3478     
3479     JITCompiler::Jump isCell = m_jit.branchIfCell(valueRegs);
3480     moveFalseTo(resultGPR);
3481     
3482     JITCompiler::Jump done = m_jit.jump();
3483     
3484     isCell.link(&m_jit);
3485     
3486     JITCompiler::Jump slowCase = m_jit.branchIfNotCell(prototypeRegs);
3487     
3488     compileInstanceOfForCells(node, valueRegs, prototypeRegs, resultGPR, scratchGPR, InvalidGPRReg, slowCase);
3489     
3490     done.link(&m_jit);
3491     blessedBooleanResult(resultGPR, node);
3492     return;
3493 }
3494
3495 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3496 void SpeculativeJIT::emitUntypedBitOp(Node* node)
3497 {
3498     Edge& leftChild = node->child1();
3499     Edge& rightChild = node->child2();
3500
3501     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3502         JSValueOperand left(this, leftChild);
3503         JSValueOperand right(this, rightChild);
3504         JSValueRegs leftRegs = left.jsValueRegs();
3505         JSValueRegs rightRegs = right.jsValueRegs();
3506
3507         flushRegisters();
3508         JSValueRegsFlushedCallResult result(this);
3509         JSValueRegs resultRegs = result.regs();
3510         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3511         m_jit.exceptionCheck();
3512
3513         jsValueResult(resultRegs, node);
3514         return;
3515     }
3516
3517     std::optional<JSValueOperand> left;
3518     std::optional<JSValueOperand> right;
3519
3520     JSValueRegs leftRegs;
3521     JSValueRegs rightRegs;
3522
3523 #if USE(JSVALUE64)
3524     GPRTemporary result(this);
3525     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3526     GPRTemporary scratch(this);
3527     GPRReg scratchGPR = scratch.gpr();
3528 #else
3529     GPRTemporary resultTag(this);
3530     GPRTemporary resultPayload(this);
3531     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3532     GPRReg scratchGPR = resultTag.gpr();
3533 #endif
3534
3535     SnippetOperand leftOperand;
3536     SnippetOperand rightOperand;
3537
3538     // The snippet generator does not support both operands being constant. If the left
3539     // operand is already const, we'll ignore the right operand's constness.
3540     if (leftChild->isInt32Constant())
3541         leftOperand.setConstInt32(leftChild->asInt32());
3542     else if (rightChild->isInt32Constant())
3543         rightOperand.setConstInt32(rightChild->asInt32());
3544
3545     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3546
3547     if (!leftOperand.isConst()) {
3548         left.emplace(this, leftChild);
3549         leftRegs = left->jsValueRegs();
3550     }
3551     if (!rightOperand.isConst()) {
3552         right.emplace(this, rightChild);
3553         rightRegs = right->jsValueRegs();
3554     }
3555
3556     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3557     gen.generateFastPath(m_jit);
3558
3559     ASSERT(gen.didEmitFastPath());
3560     gen.endJumpList().append(m_jit.jump());
3561
3562     gen.slowPathJumpList().link(&m_jit);
3563     silentSpillAllRegisters(resultRegs);
3564
3565     if (leftOperand.isConst()) {
3566         leftRegs = resultRegs;
3567         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3568     } else if (rightOperand.isConst()) {
3569         rightRegs = resultRegs;
3570         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3571     }
3572
3573     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3574
3575     silentFillAllRegisters();
3576     m_jit.exceptionCheck();
3577
3578     gen.endJumpList().link(&m_jit);
3579     jsValueResult(resultRegs, node);
3580 }
3581
3582 void SpeculativeJIT::compileBitwiseOp(Node* node)
3583 {
3584     NodeType op = node->op();
3585     Edge& leftChild = node->child1();
3586     Edge& rightChild = node->child2();
3587
3588     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3589         switch (op) {
3590         case BitAnd:
3591             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3592             return;
3593         case BitOr:
3594             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3595             return;
3596         case BitXor:
3597             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3598             return;
3599         default:
3600             RELEASE_ASSERT_NOT_REACHED();
3601         }
3602     }
3603
3604     if (leftChild->isInt32Constant()) {
3605         SpeculateInt32Operand op2(this, rightChild);
3606         GPRTemporary result(this, Reuse, op2);
3607
3608         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3609
3610         int32Result(result.gpr(), node);
3611
3612     } else if (rightChild->isInt32Constant()) {
3613         SpeculateInt32Operand op1(this, leftChild);
3614         GPRTemporary result(this, Reuse, op1);
3615
3616         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3617
3618         int32Result(result.gpr(), node);
3619
3620     } else {
3621         SpeculateInt32Operand op1(this, leftChild);
3622         SpeculateInt32Operand op2(this, rightChild);
3623         GPRTemporary result(this, Reuse, op1, op2);
3624         
3625         GPRReg reg1 = op1.gpr();
3626         GPRReg reg2 = op2.gpr();
3627         bitOp(op, reg1, reg2, result.gpr());
3628         
3629         int32Result(result.gpr(), node);
3630     }
3631 }
3632
3633 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3634 {
3635     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3636         ? operationValueBitRShift : operationValueBitURShift;
3637     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3638         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3639
3640     Edge& leftChild = node->child1();
3641     Edge& rightChild = node->child2();
3642
3643     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3644         JSValueOperand left(this, leftChild);
3645         JSValueOperand right(this, rightChild);
3646         JSValueRegs leftRegs = left.jsValueRegs();
3647         JSValueRegs rightRegs = right.jsValueRegs();
3648
3649         flushRegisters();
3650         JSValueRegsFlushedCallResult result(this);
3651         JSValueRegs resultRegs = result.regs();
3652         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3653         m_jit.exceptionCheck();
3654
3655         jsValueResult(resultRegs, node);
3656         return;
3657     }
3658
3659     std::optional<JSValueOperand> left;
3660     std::optional<JSValueOperand> right;
3661
3662     JSValueRegs leftRegs;
3663     JSValueRegs rightRegs;
3664
3665     FPRTemporary leftNumber(this);
3666     FPRReg leftFPR = leftNumber.fpr();
3667
3668 #if USE(JSVALUE64)
3669     GPRTemporary result(this);
3670     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3671     GPRTemporary scratch(this);
3672     GPRReg scratchGPR = scratch.gpr();
3673     FPRReg scratchFPR = InvalidFPRReg;
3674 #else
3675     GPRTemporary resultTag(this);
3676     GPRTemporary resultPayload(this);
3677     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3678     GPRReg scratchGPR = resultTag.gpr();
3679     FPRTemporary fprScratch(this);
3680     FPRReg scratchFPR = fprScratch.fpr();
3681 #endif
3682
3683     SnippetOperand leftOperand;
3684     SnippetOperand rightOperand;
3685
3686     // The snippet generator does not support both operands being constant. If the left
3687     // operand is already const, we'll ignore the right operand's constness.
3688     if (leftChild->isInt32Constant())
3689         leftOperand.setConstInt32(leftChild->asInt32());
3690     else if (rightChild->isInt32Constant())
3691         rightOperand.setConstInt32(rightChild->asInt32());
3692
3693     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3694
3695     if (!leftOperand.isConst()) {
3696         left.emplace(this, leftChild);
3697         leftRegs = left->jsValueRegs();
3698     }
3699     if (!rightOperand.isConst()) {
3700         right.emplace(this, rightChild);
3701         rightRegs = right->jsValueRegs();
3702     }
3703
3704     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3705         leftFPR, scratchGPR, scratchFPR, shiftType);
3706     gen.generateFastPath(m_jit);
3707
3708     ASSERT(gen.didEmitFastPath());
3709     gen.endJumpList().append(m_jit.jump());
3710
3711     gen.slowPathJumpList().link(&m_jit);
3712     silentSpillAllRegisters(resultRegs);
3713
3714     if (leftOperand.isConst()) {
3715         leftRegs = resultRegs;
3716         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3717     } else if (rightOperand.isConst()) {
3718         rightRegs = resultRegs;
3719         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3720     }
3721
3722     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3723
3724     silentFillAllRegisters();
3725     m_jit.exceptionCheck();
3726
3727     gen.endJumpList().link(&m_jit);
3728     jsValueResult(resultRegs, node);
3729     return;
3730 }
3731
3732 void SpeculativeJIT::compileShiftOp(Node* node)
3733 {
3734     NodeType op = node->op();
3735     Edge& leftChild = node->child1();
3736     Edge& rightChild = node->child2();
3737
3738     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3739         switch (op) {
3740         case BitLShift:
3741             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3742             return;
3743         case BitRShift:
3744         case BitURShift:
3745             emitUntypedRightShiftBitOp(node);
3746             return;
3747         default:
3748             RELEASE_ASSERT_NOT_REACHED();
3749         }
3750     }
3751
3752     if (rightChild->isInt32Constant()) {
3753         SpeculateInt32Operand op1(this, leftChild);
3754         GPRTemporary result(this, Reuse, op1);
3755
3756         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3757
3758         int32Result(result.gpr(), node);
3759     } else {
3760         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3761         SpeculateInt32Operand op1(this, leftChild);
3762         SpeculateInt32Operand op2(this, rightChild);
3763         GPRTemporary result(this, Reuse, op1);
3764
3765         GPRReg reg1 = op1.gpr();
3766         GPRReg reg2 = op2.gpr();
3767         shiftOp(op, reg1, reg2, result.gpr());
3768
3769         int32Result(result.gpr(), node);
3770     }
3771 }
3772
3773 void SpeculativeJIT::compileValueAdd(Node* node)
3774 {
3775     Edge& leftChild = node->child1();
3776     Edge& rightChild = node->child2();
3777
3778     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3779         JSValueOperand left(this, leftChild);
3780         JSValueOperand right(this, rightChild);
3781         JSValueRegs leftRegs = left.jsValueRegs();
3782         JSValueRegs rightRegs = right.jsValueRegs();
3783
3784         flushRegisters();
3785         JSValueRegsFlushedCallResult result(this);
3786         JSValueRegs resultRegs = result.regs();
3787         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3788         m_jit.exceptionCheck();
3789     
3790         jsValueResult(resultRegs, node);
3791         return;
3792     }
3793
3794 #if USE(JSVALUE64)
3795     bool needsScratchGPRReg = true;
3796     bool needsScratchFPRReg = false;
3797 #else
3798     bool needsScratchGPRReg = true;
3799     bool needsScratchFPRReg = true;
3800 #endif
3801
3802     CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3803     ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(node->origin.semantic.bytecodeIndex);
3804     Instruction* instruction = &baselineCodeBlock->instructions()[node->origin.semantic.bytecodeIndex];
3805     JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile, instruction);
3806     auto repatchingFunction = operationValueAddOptimize;
3807     auto nonRepatchingFunction = operationValueAdd;
3808     
3809     compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3810 }
3811
3812 template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3813 void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3814 {
3815     Edge& leftChild = node->child1();
3816     Edge& rightChild = node->child2();
3817
3818     std::optional<JSValueOperand> left;
3819     std::optional<JSValueOperand> right;
3820
3821     JSValueRegs leftRegs;
3822     JSValueRegs rightRegs;
3823
3824     FPRTemporary leftNumber(this);
3825     FPRTemporary rightNumber(this);
3826     FPRReg leftFPR = leftNumber.fpr();
3827     FPRReg rightFPR = rightNumber.fpr();
3828
3829     GPRReg scratchGPR = InvalidGPRReg;
3830     FPRReg scratchFPR = InvalidFPRReg;
3831
3832     std::optional<FPRTemporary> fprScratch;
3833     if (needsScratchFPRReg) {
3834         fprScratch.emplace(this);
3835         scratchFPR = fprScratch->fpr();
3836     }
3837
3838 #if USE(JSVALUE64)
3839     std::optional<GPRTemporary> gprScratch;
3840     if (needsScratchGPRReg) {
3841         gprScratch.emplace(this);
3842         scratchGPR = gprScratch->gpr();
3843     }
3844     GPRTemporary result(this);
3845     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3846 #else
3847     GPRTemporary resultTag(this);
3848     GPRTemporary resultPayload(this);
3849     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3850     if (needsScratchGPRReg)
3851         scratchGPR = resultRegs.tagGPR();
3852 #endif
3853
3854     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3855     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3856
3857     // The snippet generator does not support both operands being constant. If the left
3858     // operand is already const, we'll ignore the right operand's constness.
3859     if (leftChild->isInt32Constant())
3860         leftOperand.setConstInt32(leftChild->asInt32());
3861     else if (rightChild->isInt32Constant())
3862         rightOperand.setConstInt32(rightChild->asInt32());
3863
3864     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3865     ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
3866
3867     if (!Generator::isLeftOperandValidConstant(leftOperand)) {
3868         left.emplace(this, leftChild);
3869         leftRegs = left->jsValueRegs();
3870     }
3871     if (!Generator::isRightOperandValidConstant(rightOperand)) {
3872         right.emplace(this, rightChild);
3873         rightRegs = right->jsValueRegs();
3874     }
3875
3876 #if ENABLE(MATH_IC_STATS)
3877     auto inlineStart = m_jit.label();
3878 #endif
3879
3880     Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
3881     mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
3882
3883     bool shouldEmitProfiling = false;
3884     bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
3885     if (generatedInline) {
3886         ASSERT(!addICGenerationState->slowPathJumps.empty());
3887
3888         Vector<SilentRegisterSavePlan> savePlans;
3889         silentSpillAllRegistersImpl(false, savePlans, resultRegs);
3890
3891         auto done = m_jit.label();
3892
3893         addSlowPathGenerator([=, savePlans = WTFMove(savePlans)] () {
3894             addICGenerationState->slowPathJumps.link(&m_jit);
3895             addICGenerationState->slowPathStart = m_jit.label();
3896 #if ENABLE(MATH_IC_STATS)
3897             auto slowPathStart = m_jit.label();
3898 #endif
3899
3900             silentSpill(savePlans);
3901
3902             auto innerLeftRegs = leftRegs;
3903             auto innerRightRegs = rightRegs;
3904             if (Generator::isLeftOperandValidConstant(leftOperand)) {
3905                 innerLeftRegs = resultRegs;
3906                 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
3907             } else if (Generator::isRightOperandValidConstant(rightOperand)) {
3908                 innerRightRegs = resultRegs;
3909                 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
3910             }
3911
3912             if (addICGenerationState->shouldSlowPathRepatch)
3913                 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
3914             else
3915                 addICGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, innerLeftRegs, innerRightRegs);
3916
3917             silentFill(savePlans);
3918             m_jit.exceptionCheck();
3919             m_jit.jump().linkTo(done, &m_jit);
3920
3921             m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
3922                 mathIC->finalizeInlineCode(*addICGenerationState, linkBuffer);
3923             });
3924
3925 #if ENABLE(MATH_