Renaming SpecInt32, SpecInt52, MachineInt to SpecInt32Only, SpecInt52Only, AnyInt.
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGSpeculativeJIT.cpp
1 /*
2  * Copyright (C) 2011-2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "DFGSpeculativeJIT.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "BinarySwitch.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGArrayifySlowPathGenerator.h"
34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36 #include "DFGMayExit.h"
37 #include "DFGOSRExitFuzz.h"
38 #include "DFGSaneStringGetByValSlowPathGenerator.h"
39 #include "DFGSlowPathGenerator.h"
40 #include "DirectArguments.h"
41 #include "JITAddGenerator.h"
42 #include "JITBitAndGenerator.h"
43 #include "JITBitOrGenerator.h"
44 #include "JITBitXorGenerator.h"
45 #include "JITDivGenerator.h"
46 #include "JITLeftShiftGenerator.h"
47 #include "JITMulGenerator.h"
48 #include "JITRightShiftGenerator.h"
49 #include "JITSubGenerator.h"
50 #include "JSCInlines.h"
51 #include "JSEnvironmentRecord.h"
52 #include "JSGeneratorFunction.h"
53 #include "JSLexicalEnvironment.h"
54 #include "LinkBuffer.h"
55 #include "RegExpConstructor.h"
56 #include "ScopedArguments.h"
57 #include "ScratchRegisterAllocator.h"
58 #include "WriteBarrierBuffer.h"
59 #include <wtf/MathExtras.h>
60
61 namespace JSC { namespace DFG {
62
63 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
64     : m_compileOkay(true)
65     , m_jit(jit)
66     , m_currentNode(0)
67     , m_lastGeneratedNode(LastNodeType)
68     , m_indexInBlock(0)
69     , m_generationInfo(m_jit.graph().frameRegisterCount())
70     , m_state(m_jit.graph())
71     , m_interpreter(m_jit.graph(), m_state)
72     , m_stream(&jit.jitCode()->variableEventStream)
73     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
74 {
75 }
76
77 SpeculativeJIT::~SpeculativeJIT()
78 {
79 }
80
81 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, Structure* structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
82 {
83     IndexingType indexingType = structure->indexingType();
84     bool hasIndexingHeader = hasIndexedProperties(indexingType);
85
86     unsigned inlineCapacity = structure->inlineCapacity();
87     unsigned outOfLineCapacity = structure->outOfLineCapacity();
88     
89     GPRTemporary scratch(this);
90     GPRTemporary scratch2(this);
91     GPRReg scratchGPR = scratch.gpr();
92     GPRReg scratch2GPR = scratch2.gpr();
93
94     ASSERT(vectorLength >= numElements);
95     vectorLength = std::max(BASE_VECTOR_LEN, vectorLength);
96     
97     JITCompiler::JumpList slowCases;
98
99     size_t size = 0;
100     if (hasIndexingHeader)
101         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
102     size += outOfLineCapacity * sizeof(JSValue);
103
104     if (size) {
105         slowCases.append(
106             emitAllocateBasicStorage(TrustedImm32(size), storageGPR));
107         if (hasIndexingHeader)
108             m_jit.subPtr(TrustedImm32(vectorLength * sizeof(JSValue)), storageGPR);
109         else
110             m_jit.addPtr(TrustedImm32(sizeof(IndexingHeader)), storageGPR);
111     } else
112         m_jit.move(TrustedImmPtr(0), storageGPR);
113
114     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
115     MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize);
116     m_jit.move(TrustedImmPtr(allocatorPtr), scratchGPR);
117     emitAllocateJSObject(resultGPR, scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
118
119     if (hasIndexingHeader)
120         m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
121
122     // I want a slow path that also loads out the storage pointer, and that's
123     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
124     // of work for a very small piece of functionality. :-/
125     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
126         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
127         structure, vectorLength));
128
129     if (hasDouble(structure->indexingType()) && numElements < vectorLength) {
130 #if USE(JSVALUE64)
131         m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
132         for (unsigned i = numElements; i < vectorLength; ++i)
133             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
134 #else
135         EncodedValueDescriptor value;
136         value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
137         for (unsigned i = numElements; i < vectorLength; ++i) {
138             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
139             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
140         }
141 #endif
142     }
143     
144     if (hasIndexingHeader)
145         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
146 }
147
148 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
149 {
150     if (inlineCallFrame && !inlineCallFrame->isVarargs())
151         m_jit.move(TrustedImm32(inlineCallFrame->arguments.size() - !includeThis), lengthGPR);
152     else {
153         VirtualRegister argumentCountRegister;
154         if (!inlineCallFrame)
155             argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
156         else
157             argumentCountRegister = inlineCallFrame->argumentCountRegister;
158         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
159         if (!includeThis)
160             m_jit.sub32(TrustedImm32(1), lengthGPR);
161     }
162 }
163
164 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
165 {
166     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
167 }
168
169 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
170 {
171     if (origin.inlineCallFrame) {
172         if (origin.inlineCallFrame->isClosureCall) {
173             m_jit.loadPtr(
174                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
175                 calleeGPR);
176         } else {
177             m_jit.move(
178                 TrustedImmPtr(origin.inlineCallFrame->calleeRecovery.constant().asCell()),
179                 calleeGPR);
180         }
181     } else
182         m_jit.loadPtr(JITCompiler::addressFor(JSStack::Callee), calleeGPR);
183 }
184
185 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
186 {
187     m_jit.addPtr(
188         TrustedImm32(
189             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
190         GPRInfo::callFrameRegister, startGPR);
191 }
192
193 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
194 {
195     if (!doOSRExitFuzzing())
196         return MacroAssembler::Jump();
197     
198     MacroAssembler::Jump result;
199     
200     m_jit.pushToSave(GPRInfo::regT0);
201     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
202     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
203     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
204     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
205     unsigned at = Options::fireOSRExitFuzzAt();
206     if (at || atOrAfter) {
207         unsigned threshold;
208         MacroAssembler::RelationalCondition condition;
209         if (atOrAfter) {
210             threshold = atOrAfter;
211             condition = MacroAssembler::Below;
212         } else {
213             threshold = at;
214             condition = MacroAssembler::NotEqual;
215         }
216         MacroAssembler::Jump ok = m_jit.branch32(
217             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
218         m_jit.popToRestore(GPRInfo::regT0);
219         result = m_jit.jump();
220         ok.link(&m_jit);
221     }
222     m_jit.popToRestore(GPRInfo::regT0);
223     
224     return result;
225 }
226
227 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
228 {
229     if (!m_compileOkay)
230         return;
231     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
232     if (fuzzJump.isSet()) {
233         JITCompiler::JumpList jumpsToFail;
234         jumpsToFail.append(fuzzJump);
235         jumpsToFail.append(jumpToFail);
236         m_jit.appendExitInfo(jumpsToFail);
237     } else
238         m_jit.appendExitInfo(jumpToFail);
239     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size()));
240 }
241
242 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
243 {
244     if (!m_compileOkay)
245         return;
246     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
247     if (fuzzJump.isSet()) {
248         JITCompiler::JumpList myJumpsToFail;
249         myJumpsToFail.append(jumpsToFail);
250         myJumpsToFail.append(fuzzJump);
251         m_jit.appendExitInfo(myJumpsToFail);
252     } else
253         m_jit.appendExitInfo(jumpsToFail);
254     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size()));
255 }
256
257 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
258 {
259     if (!m_compileOkay)
260         return OSRExitJumpPlaceholder();
261     unsigned index = m_jit.jitCode()->osrExit.size();
262     m_jit.appendExitInfo();
263     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size()));
264     return OSRExitJumpPlaceholder(index);
265 }
266
267 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
268 {
269     return speculationCheck(kind, jsValueSource, nodeUse.node());
270 }
271
272 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
273 {
274     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
275 }
276
277 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
278 {
279     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
280 }
281
282 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
283 {
284     if (!m_compileOkay)
285         return;
286     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
287     m_jit.appendExitInfo(jumpToFail);
288     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size(), recoveryIndex));
289 }
290
291 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
292 {
293     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
294 }
295
296 void SpeculativeJIT::emitInvalidationPoint(Node* node)
297 {
298     if (!m_compileOkay)
299         return;
300     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
301     m_jit.jitCode()->appendOSRExit(OSRExit(
302         UncountableInvalidation, JSValueSource(),
303         m_jit.graph().methodOfGettingAValueProfileFor(node),
304         this, m_stream->size()));
305     info.m_replacementSource = m_jit.watchpointLabel();
306     ASSERT(info.m_replacementSource.isSet());
307     noResult(node);
308 }
309
310 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
311 {
312     if (!m_compileOkay)
313         return;
314     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
315     m_compileOkay = false;
316     if (verboseCompilationEnabled())
317         dataLog("Bailing compilation.\n");
318 }
319
320 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
321 {
322     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
323 }
324
325 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
326 {
327     ASSERT(needsTypeCheck(edge, typesPassedThrough));
328     m_interpreter.filter(edge, typesPassedThrough);
329     speculationCheck(exitKind, source, edge.node(), jumpToFail);
330 }
331
332 RegisterSet SpeculativeJIT::usedRegisters()
333 {
334     RegisterSet result;
335     
336     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
337         GPRReg gpr = GPRInfo::toRegister(i);
338         if (m_gprs.isInUse(gpr))
339             result.set(gpr);
340     }
341     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
342         FPRReg fpr = FPRInfo::toRegister(i);
343         if (m_fprs.isInUse(fpr))
344             result.set(fpr);
345     }
346     
347     result.merge(RegisterSet::stubUnavailableRegisters());
348     
349     return result;
350 }
351
352 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
353 {
354     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
355 }
356
357 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
358 {
359     for (unsigned i = 0; i < m_slowPathGenerators.size(); ++i) {
360         pcToCodeOriginMapBuilder.appendItem(m_jit.label(), m_slowPathGenerators[i]->origin().semantic);
361         m_slowPathGenerators[i]->generate(this);
362     }
363 }
364
365 // On Windows we need to wrap fmod; on other platforms we can call it directly.
366 // On ARMv7 we assert that all function pointers have to low bit set (point to thumb code).
367 #if CALLING_CONVENTION_IS_STDCALL || CPU(ARM_THUMB2)
368 static double JIT_OPERATION fmodAsDFGOperation(double x, double y)
369 {
370     return fmod(x, y);
371 }
372 #else
373 #define fmodAsDFGOperation fmod
374 #endif
375
376 void SpeculativeJIT::clearGenerationInfo()
377 {
378     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
379         m_generationInfo[i] = GenerationInfo();
380     m_gprs = RegisterBank<GPRInfo>();
381     m_fprs = RegisterBank<FPRInfo>();
382 }
383
384 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
385 {
386     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
387     Node* node = info.node();
388     DataFormat registerFormat = info.registerFormat();
389     ASSERT(registerFormat != DataFormatNone);
390     ASSERT(registerFormat != DataFormatDouble);
391         
392     SilentSpillAction spillAction;
393     SilentFillAction fillAction;
394         
395     if (!info.needsSpill())
396         spillAction = DoNothingForSpill;
397     else {
398 #if USE(JSVALUE64)
399         ASSERT(info.gpr() == source);
400         if (registerFormat == DataFormatInt32)
401             spillAction = Store32Payload;
402         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
403             spillAction = StorePtr;
404         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
405             spillAction = Store64;
406         else {
407             ASSERT(registerFormat & DataFormatJS);
408             spillAction = Store64;
409         }
410 #elif USE(JSVALUE32_64)
411         if (registerFormat & DataFormatJS) {
412             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
413             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
414         } else {
415             ASSERT(info.gpr() == source);
416             spillAction = Store32Payload;
417         }
418 #endif
419     }
420         
421     if (registerFormat == DataFormatInt32) {
422         ASSERT(info.gpr() == source);
423         ASSERT(isJSInt32(info.registerFormat()));
424         if (node->hasConstant()) {
425             ASSERT(node->isInt32Constant());
426             fillAction = SetInt32Constant;
427         } else
428             fillAction = Load32Payload;
429     } else if (registerFormat == DataFormatBoolean) {
430 #if USE(JSVALUE64)
431         RELEASE_ASSERT_NOT_REACHED();
432 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
433         fillAction = DoNothingForFill;
434 #endif
435 #elif USE(JSVALUE32_64)
436         ASSERT(info.gpr() == source);
437         if (node->hasConstant()) {
438             ASSERT(node->isBooleanConstant());
439             fillAction = SetBooleanConstant;
440         } else
441             fillAction = Load32Payload;
442 #endif
443     } else if (registerFormat == DataFormatCell) {
444         ASSERT(info.gpr() == source);
445         if (node->hasConstant()) {
446             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
447             node->asCell(); // To get the assertion.
448             fillAction = SetCellConstant;
449         } else {
450 #if USE(JSVALUE64)
451             fillAction = LoadPtr;
452 #else
453             fillAction = Load32Payload;
454 #endif
455         }
456     } else if (registerFormat == DataFormatStorage) {
457         ASSERT(info.gpr() == source);
458         fillAction = LoadPtr;
459     } else if (registerFormat == DataFormatInt52) {
460         if (node->hasConstant())
461             fillAction = SetInt52Constant;
462         else if (info.spillFormat() == DataFormatInt52)
463             fillAction = Load64;
464         else if (info.spillFormat() == DataFormatStrictInt52)
465             fillAction = Load64ShiftInt52Left;
466         else if (info.spillFormat() == DataFormatNone)
467             fillAction = Load64;
468         else {
469             RELEASE_ASSERT_NOT_REACHED();
470 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
471             fillAction = Load64; // Make GCC happy.
472 #endif
473         }
474     } else if (registerFormat == DataFormatStrictInt52) {
475         if (node->hasConstant())
476             fillAction = SetStrictInt52Constant;
477         else if (info.spillFormat() == DataFormatInt52)
478             fillAction = Load64ShiftInt52Right;
479         else if (info.spillFormat() == DataFormatStrictInt52)
480             fillAction = Load64;
481         else if (info.spillFormat() == DataFormatNone)
482             fillAction = Load64;
483         else {
484             RELEASE_ASSERT_NOT_REACHED();
485 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
486             fillAction = Load64; // Make GCC happy.
487 #endif
488         }
489     } else {
490         ASSERT(registerFormat & DataFormatJS);
491 #if USE(JSVALUE64)
492         ASSERT(info.gpr() == source);
493         if (node->hasConstant()) {
494             if (node->isCellConstant())
495                 fillAction = SetTrustedJSConstant;
496             else
497                 fillAction = SetJSConstant;
498         } else if (info.spillFormat() == DataFormatInt32) {
499             ASSERT(registerFormat == DataFormatJSInt32);
500             fillAction = Load32PayloadBoxInt;
501         } else
502             fillAction = Load64;
503 #else
504         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
505         if (node->hasConstant())
506             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
507         else if (info.payloadGPR() == source)
508             fillAction = Load32Payload;
509         else { // Fill the Tag
510             switch (info.spillFormat()) {
511             case DataFormatInt32:
512                 ASSERT(registerFormat == DataFormatJSInt32);
513                 fillAction = SetInt32Tag;
514                 break;
515             case DataFormatCell:
516                 ASSERT(registerFormat == DataFormatJSCell);
517                 fillAction = SetCellTag;
518                 break;
519             case DataFormatBoolean:
520                 ASSERT(registerFormat == DataFormatJSBoolean);
521                 fillAction = SetBooleanTag;
522                 break;
523             default:
524                 fillAction = Load32Tag;
525                 break;
526             }
527         }
528 #endif
529     }
530         
531     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
532 }
533     
534 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
535 {
536     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
537     Node* node = info.node();
538     ASSERT(info.registerFormat() == DataFormatDouble);
539
540     SilentSpillAction spillAction;
541     SilentFillAction fillAction;
542         
543     if (!info.needsSpill())
544         spillAction = DoNothingForSpill;
545     else {
546         ASSERT(!node->hasConstant());
547         ASSERT(info.spillFormat() == DataFormatNone);
548         ASSERT(info.fpr() == source);
549         spillAction = StoreDouble;
550     }
551         
552 #if USE(JSVALUE64)
553     if (node->hasConstant()) {
554         node->asNumber(); // To get the assertion.
555         fillAction = SetDoubleConstant;
556     } else {
557         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
558         fillAction = LoadDouble;
559     }
560 #elif USE(JSVALUE32_64)
561     ASSERT(info.registerFormat() == DataFormatDouble);
562     if (node->hasConstant()) {
563         node->asNumber(); // To get the assertion.
564         fillAction = SetDoubleConstant;
565     } else
566         fillAction = LoadDouble;
567 #endif
568
569     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
570 }
571     
572 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
573 {
574     switch (plan.spillAction()) {
575     case DoNothingForSpill:
576         break;
577     case Store32Tag:
578         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
579         break;
580     case Store32Payload:
581         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
582         break;
583     case StorePtr:
584         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
585         break;
586 #if USE(JSVALUE64)
587     case Store64:
588         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
589         break;
590 #endif
591     case StoreDouble:
592         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
593         break;
594     default:
595         RELEASE_ASSERT_NOT_REACHED();
596     }
597 }
598     
599 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan, GPRReg canTrample)
600 {
601 #if USE(JSVALUE32_64)
602     UNUSED_PARAM(canTrample);
603 #endif
604     switch (plan.fillAction()) {
605     case DoNothingForFill:
606         break;
607     case SetInt32Constant:
608         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
609         break;
610 #if USE(JSVALUE64)
611     case SetInt52Constant:
612         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
613         break;
614     case SetStrictInt52Constant:
615         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
616         break;
617 #endif // USE(JSVALUE64)
618     case SetBooleanConstant:
619         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
620         break;
621     case SetCellConstant:
622         m_jit.move(TrustedImmPtr(plan.node()->asCell()), plan.gpr());
623         break;
624 #if USE(JSVALUE64)
625     case SetTrustedJSConstant:
626         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
627         break;
628     case SetJSConstant:
629         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
630         break;
631     case SetDoubleConstant:
632         m_jit.move(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), canTrample);
633         m_jit.move64ToDouble(canTrample, plan.fpr());
634         break;
635     case Load32PayloadBoxInt:
636         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
637         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
638         break;
639     case Load32PayloadConvertToInt52:
640         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
641         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
642         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
643         break;
644     case Load32PayloadSignExtend:
645         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
646         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
647         break;
648 #else
649     case SetJSConstantTag:
650         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
651         break;
652     case SetJSConstantPayload:
653         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
654         break;
655     case SetInt32Tag:
656         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
657         break;
658     case SetCellTag:
659         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
660         break;
661     case SetBooleanTag:
662         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
663         break;
664     case SetDoubleConstant:
665         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
666         break;
667 #endif
668     case Load32Tag:
669         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
670         break;
671     case Load32Payload:
672         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
673         break;
674     case LoadPtr:
675         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
676         break;
677 #if USE(JSVALUE64)
678     case Load64:
679         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
680         break;
681     case Load64ShiftInt52Right:
682         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
683         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
684         break;
685     case Load64ShiftInt52Left:
686         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
687         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
688         break;
689 #endif
690     case LoadDouble:
691         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
692         break;
693     default:
694         RELEASE_ASSERT_NOT_REACHED();
695     }
696 }
697     
698 JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, IndexingType shape)
699 {
700     switch (arrayMode.arrayClass()) {
701     case Array::OriginalArray: {
702         CRASH();
703 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
704         JITCompiler::Jump result; // I already know that VC++ takes unkindly to the expression "return Jump()", so I'm doing it this way in anticipation of someone eventually using VC++ to compile the DFG.
705         return result;
706 #endif
707     }
708         
709     case Array::Array:
710         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
711         return m_jit.branch32(
712             MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape));
713         
714     case Array::NonArray:
715     case Array::OriginalNonArray:
716         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
717         return m_jit.branch32(
718             MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
719         
720     case Array::PossiblyArray:
721         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
722         return m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
723     }
724     
725     RELEASE_ASSERT_NOT_REACHED();
726     return JITCompiler::Jump();
727 }
728
729 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
730 {
731     JITCompiler::JumpList result;
732     
733     switch (arrayMode.type()) {
734     case Array::Int32:
735         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, Int32Shape);
736
737     case Array::Double:
738         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, DoubleShape);
739
740     case Array::Contiguous:
741         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, ContiguousShape);
742
743     case Array::Undecided:
744         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, UndecidedShape);
745
746     case Array::ArrayStorage:
747     case Array::SlowPutArrayStorage: {
748         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
749         
750         if (arrayMode.isJSArray()) {
751             if (arrayMode.isSlowPut()) {
752                 result.append(
753                     m_jit.branchTest32(
754                         MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
755                 m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
756                 m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
757                 result.append(
758                     m_jit.branch32(
759                         MacroAssembler::Above, tempGPR,
760                         TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
761                 break;
762             }
763             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
764             result.append(
765                 m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | ArrayStorageShape)));
766             break;
767         }
768         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
769         if (arrayMode.isSlowPut()) {
770             m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
771             result.append(
772                 m_jit.branch32(
773                     MacroAssembler::Above, tempGPR,
774                     TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
775             break;
776         }
777         result.append(
778             m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape)));
779         break;
780     }
781     default:
782         CRASH();
783         break;
784     }
785     
786     return result;
787 }
788
789 void SpeculativeJIT::checkArray(Node* node)
790 {
791     ASSERT(node->arrayMode().isSpecific());
792     ASSERT(!node->arrayMode().doesConversion());
793     
794     SpeculateCellOperand base(this, node->child1());
795     GPRReg baseReg = base.gpr();
796     
797     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
798         noResult(m_currentNode);
799         return;
800     }
801     
802     const ClassInfo* expectedClassInfo = 0;
803     
804     switch (node->arrayMode().type()) {
805     case Array::AnyTypedArray:
806     case Array::String:
807         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
808         break;
809     case Array::Int32:
810     case Array::Double:
811     case Array::Contiguous:
812     case Array::Undecided:
813     case Array::ArrayStorage:
814     case Array::SlowPutArrayStorage: {
815         GPRTemporary temp(this);
816         GPRReg tempGPR = temp.gpr();
817         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeOffset()), tempGPR);
818         speculationCheck(
819             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
820             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
821         
822         noResult(m_currentNode);
823         return;
824     }
825     case Array::DirectArguments:
826         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
827         noResult(m_currentNode);
828         return;
829     case Array::ScopedArguments:
830         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
831         noResult(m_currentNode);
832         return;
833     default:
834         speculateCellTypeWithoutTypeFiltering(
835             node->child1(), baseReg,
836             typeForTypedArrayType(node->arrayMode().typedArrayType()));
837         noResult(m_currentNode);
838         return;
839     }
840     
841     RELEASE_ASSERT(expectedClassInfo);
842     
843     GPRTemporary temp(this);
844     GPRTemporary temp2(this);
845     m_jit.emitLoadStructure(baseReg, temp.gpr(), temp2.gpr());
846     speculationCheck(
847         BadType, JSValueSource::unboxedCell(baseReg), node,
848         m_jit.branchPtr(
849             MacroAssembler::NotEqual,
850             MacroAssembler::Address(temp.gpr(), Structure::classInfoOffset()),
851             MacroAssembler::TrustedImmPtr(expectedClassInfo)));
852     
853     noResult(m_currentNode);
854 }
855
856 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
857 {
858     ASSERT(node->arrayMode().doesConversion());
859     
860     GPRTemporary temp(this);
861     GPRTemporary structure;
862     GPRReg tempGPR = temp.gpr();
863     GPRReg structureGPR = InvalidGPRReg;
864     
865     if (node->op() != ArrayifyToStructure) {
866         GPRTemporary realStructure(this);
867         structure.adopt(realStructure);
868         structureGPR = structure.gpr();
869     }
870         
871     // We can skip all that comes next if we already have array storage.
872     MacroAssembler::JumpList slowPath;
873     
874     if (node->op() == ArrayifyToStructure) {
875         slowPath.append(m_jit.branchWeakStructure(
876             JITCompiler::NotEqual,
877             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
878             node->structure()));
879     } else {
880         m_jit.load8(
881             MacroAssembler::Address(baseReg, JSCell::indexingTypeOffset()), tempGPR);
882         
883         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
884     }
885     
886     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
887         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
888     
889     noResult(m_currentNode);
890 }
891
892 void SpeculativeJIT::arrayify(Node* node)
893 {
894     ASSERT(node->arrayMode().isSpecific());
895     
896     SpeculateCellOperand base(this, node->child1());
897     
898     if (!node->child2()) {
899         arrayify(node, base.gpr(), InvalidGPRReg);
900         return;
901     }
902     
903     SpeculateInt32Operand property(this, node->child2());
904     
905     arrayify(node, base.gpr(), property.gpr());
906 }
907
908 GPRReg SpeculativeJIT::fillStorage(Edge edge)
909 {
910     VirtualRegister virtualRegister = edge->virtualRegister();
911     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
912     
913     switch (info.registerFormat()) {
914     case DataFormatNone: {
915         if (info.spillFormat() == DataFormatStorage) {
916             GPRReg gpr = allocate();
917             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
918             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
919             info.fillStorage(*m_stream, gpr);
920             return gpr;
921         }
922         
923         // Must be a cell; fill it as a cell and then return the pointer.
924         return fillSpeculateCell(edge);
925     }
926         
927     case DataFormatStorage: {
928         GPRReg gpr = info.gpr();
929         m_gprs.lock(gpr);
930         return gpr;
931     }
932         
933     default:
934         return fillSpeculateCell(edge);
935     }
936 }
937
938 void SpeculativeJIT::useChildren(Node* node)
939 {
940     if (node->flags() & NodeHasVarArgs) {
941         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
942             if (!!m_jit.graph().m_varArgChildren[childIdx])
943                 use(m_jit.graph().m_varArgChildren[childIdx]);
944         }
945     } else {
946         Edge child1 = node->child1();
947         if (!child1) {
948             ASSERT(!node->child2() && !node->child3());
949             return;
950         }
951         use(child1);
952         
953         Edge child2 = node->child2();
954         if (!child2) {
955             ASSERT(!node->child3());
956             return;
957         }
958         use(child2);
959         
960         Edge child3 = node->child3();
961         if (!child3)
962             return;
963         use(child3);
964     }
965 }
966
967 void SpeculativeJIT::compileTryGetById(Node* node)
968 {
969     switch (node->child1().useKind()) {
970     case CellUse: {
971         SpeculateCellOperand base(this, node->child1());
972         JSValueRegsTemporary result(this, Reuse, base);
973
974         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
975         JSValueRegs resultRegs = result.regs();
976
977         base.use();
978
979         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), DontSpill, AccessType::GetPure);
980
981         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
982         break;
983     }
984
985     case UntypedUse: {
986         JSValueOperand base(this, node->child1());
987         JSValueRegsTemporary result(this, Reuse, base);
988
989         JSValueRegs baseRegs = base.jsValueRegs();
990         JSValueRegs resultRegs = result.regs();
991
992         base.use();
993
994         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
995
996         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, DontSpill, AccessType::GetPure);
997
998         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
999         break;
1000     }
1001
1002     default:
1003         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1004         break;
1005     } 
1006 }
1007
1008 void SpeculativeJIT::compileIn(Node* node)
1009 {
1010     SpeculateCellOperand base(this, node->child2());
1011     GPRReg baseGPR = base.gpr();
1012     
1013     if (JSString* string = node->child1()->dynamicCastConstant<JSString*>()) {
1014         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1015             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1016             
1017             GPRTemporary result(this);
1018             GPRReg resultGPR = result.gpr();
1019
1020             use(node->child1());
1021             
1022             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1023             MacroAssembler::Label done = m_jit.label();
1024             
1025             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1026             // we can cast it to const AtomicStringImpl* safely.
1027             auto slowPath = slowPathCall(
1028                 jump.m_jump, this, operationInOptimize,
1029                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1030                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1031             
1032             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1033             stubInfo->codeOrigin = node->origin.semantic;
1034             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1035             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1036 #if USE(JSVALUE32_64)
1037             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1038             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1039 #endif
1040             stubInfo->patch.usedRegisters = usedRegisters();
1041
1042             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1043             addSlowPathGenerator(WTFMove(slowPath));
1044
1045             base.use();
1046
1047             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1048             return;
1049         }
1050     }
1051
1052     JSValueOperand key(this, node->child1());
1053     JSValueRegs regs = key.jsValueRegs();
1054         
1055     GPRFlushedCallResult result(this);
1056     GPRReg resultGPR = result.gpr();
1057         
1058     base.use();
1059     key.use();
1060         
1061     flushRegisters();
1062     callOperation(
1063         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1064         baseGPR, regs);
1065     m_jit.exceptionCheck();
1066     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1067 }
1068
1069 void SpeculativeJIT::compileDeleteById(Node* node)
1070 {
1071     JSValueOperand value(this, node->child1());
1072     GPRFlushedCallResult result(this);
1073
1074     JSValueRegs valueRegs = value.jsValueRegs();
1075     GPRReg resultGPR = result.gpr();
1076
1077     value.use();
1078
1079     flushRegisters();
1080     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1081     m_jit.exceptionCheck();
1082
1083     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1084 }
1085
1086 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1087 {
1088     unsigned branchIndexInBlock = detectPeepHoleBranch();
1089     if (branchIndexInBlock != UINT_MAX) {
1090         Node* branchNode = m_block->at(branchIndexInBlock);
1091
1092         ASSERT(node->adjustedRefCount() == 1);
1093         
1094         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1095     
1096         m_indexInBlock = branchIndexInBlock;
1097         m_currentNode = branchNode;
1098         
1099         return true;
1100     }
1101     
1102     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1103     
1104     return false;
1105 }
1106
1107 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1108 {
1109     unsigned branchIndexInBlock = detectPeepHoleBranch();
1110     if (branchIndexInBlock != UINT_MAX) {
1111         Node* branchNode = m_block->at(branchIndexInBlock);
1112
1113         ASSERT(node->adjustedRefCount() == 1);
1114         
1115         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1116     
1117         m_indexInBlock = branchIndexInBlock;
1118         m_currentNode = branchNode;
1119         
1120         return true;
1121     }
1122     
1123     nonSpeculativeNonPeepholeStrictEq(node, invert);
1124     
1125     return false;
1126 }
1127
1128 static const char* dataFormatString(DataFormat format)
1129 {
1130     // These values correspond to the DataFormat enum.
1131     const char* strings[] = {
1132         "[  ]",
1133         "[ i]",
1134         "[ d]",
1135         "[ c]",
1136         "Err!",
1137         "Err!",
1138         "Err!",
1139         "Err!",
1140         "[J ]",
1141         "[Ji]",
1142         "[Jd]",
1143         "[Jc]",
1144         "Err!",
1145         "Err!",
1146         "Err!",
1147         "Err!",
1148     };
1149     return strings[format];
1150 }
1151
1152 void SpeculativeJIT::dump(const char* label)
1153 {
1154     if (label)
1155         dataLogF("<%s>\n", label);
1156
1157     dataLogF("  gprs:\n");
1158     m_gprs.dump();
1159     dataLogF("  fprs:\n");
1160     m_fprs.dump();
1161     dataLogF("  VirtualRegisters:\n");
1162     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1163         GenerationInfo& info = m_generationInfo[i];
1164         if (info.alive())
1165             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1166         else
1167             dataLogF("    % 3d:[__][__]", i);
1168         if (info.registerFormat() == DataFormatDouble)
1169             dataLogF(":fpr%d\n", info.fpr());
1170         else if (info.registerFormat() != DataFormatNone
1171 #if USE(JSVALUE32_64)
1172             && !(info.registerFormat() & DataFormatJS)
1173 #endif
1174             ) {
1175             ASSERT(info.gpr() != InvalidGPRReg);
1176             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1177         } else
1178             dataLogF("\n");
1179     }
1180     if (label)
1181         dataLogF("</%s>\n", label);
1182 }
1183
1184 GPRTemporary::GPRTemporary()
1185     : m_jit(0)
1186     , m_gpr(InvalidGPRReg)
1187 {
1188 }
1189
1190 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1191     : m_jit(jit)
1192     , m_gpr(InvalidGPRReg)
1193 {
1194     m_gpr = m_jit->allocate();
1195 }
1196
1197 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1198     : m_jit(jit)
1199     , m_gpr(InvalidGPRReg)
1200 {
1201     m_gpr = m_jit->allocate(specific);
1202 }
1203
1204 #if USE(JSVALUE32_64)
1205 GPRTemporary::GPRTemporary(
1206     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1207     : m_jit(jit)
1208     , m_gpr(InvalidGPRReg)
1209 {
1210     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1211         m_gpr = m_jit->reuse(op1.gpr(which));
1212     else
1213         m_gpr = m_jit->allocate();
1214 }
1215 #endif // USE(JSVALUE32_64)
1216
1217 JSValueRegsTemporary::JSValueRegsTemporary() { }
1218
1219 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1220 #if USE(JSVALUE64)
1221     : m_gpr(jit)
1222 #else
1223     : m_payloadGPR(jit)
1224     , m_tagGPR(jit)
1225 #endif
1226 {
1227 }
1228
1229 #if USE(JSVALUE64)
1230 template<typename T>
1231 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1232     : m_gpr(jit, Reuse, operand)
1233 {
1234 }
1235 #else
1236 template<typename T>
1237 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1238 {
1239     if (resultWord == PayloadWord) {
1240         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1241         m_tagGPR = GPRTemporary(jit);
1242     } else {
1243         m_payloadGPR = GPRTemporary(jit);
1244         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1245     }
1246 }
1247 #endif
1248
1249 #if USE(JSVALUE64)
1250 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1251 {
1252     m_gpr = GPRTemporary(jit, Reuse, operand);
1253 }
1254 #else
1255 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1256 {
1257     if (jit->canReuse(operand.node())) {
1258         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1259         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1260     } else {
1261         m_payloadGPR = GPRTemporary(jit);
1262         m_tagGPR = GPRTemporary(jit);
1263     }
1264 }
1265 #endif
1266
1267 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1268
1269 JSValueRegs JSValueRegsTemporary::regs()
1270 {
1271 #if USE(JSVALUE64)
1272     return JSValueRegs(m_gpr.gpr());
1273 #else
1274     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1275 #endif
1276 }
1277
1278 void GPRTemporary::adopt(GPRTemporary& other)
1279 {
1280     ASSERT(!m_jit);
1281     ASSERT(m_gpr == InvalidGPRReg);
1282     ASSERT(other.m_jit);
1283     ASSERT(other.m_gpr != InvalidGPRReg);
1284     m_jit = other.m_jit;
1285     m_gpr = other.m_gpr;
1286     other.m_jit = 0;
1287     other.m_gpr = InvalidGPRReg;
1288 }
1289
1290 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1291     : m_jit(jit)
1292     , m_fpr(InvalidFPRReg)
1293 {
1294     m_fpr = m_jit->fprAllocate();
1295 }
1296
1297 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1298     : m_jit(jit)
1299     , m_fpr(InvalidFPRReg)
1300 {
1301     if (m_jit->canReuse(op1.node()))
1302         m_fpr = m_jit->reuse(op1.fpr());
1303     else
1304         m_fpr = m_jit->fprAllocate();
1305 }
1306
1307 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1308     : m_jit(jit)
1309     , m_fpr(InvalidFPRReg)
1310 {
1311     if (m_jit->canReuse(op1.node()))
1312         m_fpr = m_jit->reuse(op1.fpr());
1313     else if (m_jit->canReuse(op2.node()))
1314         m_fpr = m_jit->reuse(op2.fpr());
1315     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1316         m_fpr = m_jit->reuse(op1.fpr());
1317     else
1318         m_fpr = m_jit->fprAllocate();
1319 }
1320
1321 #if USE(JSVALUE32_64)
1322 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1323     : m_jit(jit)
1324     , m_fpr(InvalidFPRReg)
1325 {
1326     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1327         m_fpr = m_jit->reuse(op1.fpr());
1328     else
1329         m_fpr = m_jit->fprAllocate();
1330 }
1331 #endif
1332
1333 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1334 {
1335     BasicBlock* taken = branchNode->branchData()->taken.block;
1336     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1337
1338     if (taken == nextBlock()) {
1339         condition = MacroAssembler::invert(condition);
1340         std::swap(taken, notTaken);
1341     }
1342
1343     SpeculateDoubleOperand op1(this, node->child1());
1344     SpeculateDoubleOperand op2(this, node->child2());
1345     
1346     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1347     jump(notTaken);
1348 }
1349
1350 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1351 {
1352     BasicBlock* taken = branchNode->branchData()->taken.block;
1353     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1354
1355     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1356     
1357     if (taken == nextBlock()) {
1358         condition = MacroAssembler::NotEqual;
1359         BasicBlock* tmp = taken;
1360         taken = notTaken;
1361         notTaken = tmp;
1362     }
1363
1364     SpeculateCellOperand op1(this, node->child1());
1365     SpeculateCellOperand op2(this, node->child2());
1366     
1367     GPRReg op1GPR = op1.gpr();
1368     GPRReg op2GPR = op2.gpr();
1369     
1370     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1371         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1372             speculationCheck(
1373                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1374         }
1375         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1376             speculationCheck(
1377                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1378         }
1379     } else {
1380         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1381             speculationCheck(
1382                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1383                 m_jit.branchIfNotObject(op1GPR));
1384         }
1385         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1386             m_jit.branchTest8(
1387                 MacroAssembler::NonZero, 
1388                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), 
1389                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1390
1391         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1392             speculationCheck(
1393                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1394                 m_jit.branchIfNotObject(op2GPR));
1395         }
1396         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1397             m_jit.branchTest8(
1398                 MacroAssembler::NonZero, 
1399                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), 
1400                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1401     }
1402
1403     branchPtr(condition, op1GPR, op2GPR, taken);
1404     jump(notTaken);
1405 }
1406
1407 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1408 {
1409     BasicBlock* taken = branchNode->branchData()->taken.block;
1410     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1411
1412     // The branch instruction will branch to the taken block.
1413     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1414     if (taken == nextBlock()) {
1415         condition = JITCompiler::invert(condition);
1416         BasicBlock* tmp = taken;
1417         taken = notTaken;
1418         notTaken = tmp;
1419     }
1420
1421     if (node->child1()->isInt32Constant()) {
1422         int32_t imm = node->child1()->asInt32();
1423         SpeculateBooleanOperand op2(this, node->child2());
1424         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1425     } else if (node->child2()->isInt32Constant()) {
1426         SpeculateBooleanOperand op1(this, node->child1());
1427         int32_t imm = node->child2()->asInt32();
1428         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1429     } else {
1430         SpeculateBooleanOperand op1(this, node->child1());
1431         SpeculateBooleanOperand op2(this, node->child2());
1432         branch32(condition, op1.gpr(), op2.gpr(), taken);
1433     }
1434
1435     jump(notTaken);
1436 }
1437
1438 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1439 {
1440     BasicBlock* taken = branchNode->branchData()->taken.block;
1441     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1442
1443     // The branch instruction will branch to the taken block.
1444     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1445     if (taken == nextBlock()) {
1446         condition = JITCompiler::invert(condition);
1447         BasicBlock* tmp = taken;
1448         taken = notTaken;
1449         notTaken = tmp;
1450     }
1451
1452     if (node->child1()->isInt32Constant()) {
1453         int32_t imm = node->child1()->asInt32();
1454         SpeculateInt32Operand op2(this, node->child2());
1455         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1456     } else if (node->child2()->isInt32Constant()) {
1457         SpeculateInt32Operand op1(this, node->child1());
1458         int32_t imm = node->child2()->asInt32();
1459         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1460     } else {
1461         SpeculateInt32Operand op1(this, node->child1());
1462         SpeculateInt32Operand op2(this, node->child2());
1463         branch32(condition, op1.gpr(), op2.gpr(), taken);
1464     }
1465
1466     jump(notTaken);
1467 }
1468
1469 // Returns true if the compare is fused with a subsequent branch.
1470 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1471 {
1472     // Fused compare & branch.
1473     unsigned branchIndexInBlock = detectPeepHoleBranch();
1474     if (branchIndexInBlock != UINT_MAX) {
1475         Node* branchNode = m_block->at(branchIndexInBlock);
1476
1477         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1478         // so can be no intervening nodes to also reference the compare. 
1479         ASSERT(node->adjustedRefCount() == 1);
1480
1481         if (node->isBinaryUseKind(Int32Use))
1482             compilePeepHoleInt32Branch(node, branchNode, condition);
1483 #if USE(JSVALUE64)
1484         else if (node->isBinaryUseKind(Int52RepUse))
1485             compilePeepHoleInt52Branch(node, branchNode, condition);
1486 #endif // USE(JSVALUE64)
1487         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1488             // Use non-peephole comparison, for now.
1489             return false;
1490         } else if (node->isBinaryUseKind(DoubleRepUse))
1491             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1492         else if (node->op() == CompareEq) {
1493             if (node->isBinaryUseKind(BooleanUse))
1494                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1495             else if (node->isBinaryUseKind(SymbolUse))
1496                 compilePeepHoleSymbolEquality(node, branchNode);
1497             else if (node->isBinaryUseKind(ObjectUse))
1498                 compilePeepHoleObjectEquality(node, branchNode);
1499             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1500                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1501             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1502                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1503             else if (!needsTypeCheck(node->child1(), SpecOther))
1504                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1505             else if (!needsTypeCheck(node->child2(), SpecOther))
1506                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1507             else {
1508                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1509                 return true;
1510             }
1511         } else {
1512             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1513             return true;
1514         }
1515
1516         use(node->child1());
1517         use(node->child2());
1518         m_indexInBlock = branchIndexInBlock;
1519         m_currentNode = branchNode;
1520         return true;
1521     }
1522     return false;
1523 }
1524
1525 void SpeculativeJIT::noticeOSRBirth(Node* node)
1526 {
1527     if (!node->hasVirtualRegister())
1528         return;
1529     
1530     VirtualRegister virtualRegister = node->virtualRegister();
1531     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1532     
1533     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1534 }
1535
1536 void SpeculativeJIT::compileMovHint(Node* node)
1537 {
1538     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1539     
1540     Node* child = node->child1().node();
1541     noticeOSRBirth(child);
1542     
1543     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1544 }
1545
1546 void SpeculativeJIT::bail(AbortReason reason)
1547 {
1548     if (verboseCompilationEnabled())
1549         dataLog("Bailing compilation.\n");
1550     m_compileOkay = true;
1551     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1552     clearGenerationInfo();
1553 }
1554
1555 void SpeculativeJIT::compileCurrentBlock()
1556 {
1557     ASSERT(m_compileOkay);
1558     
1559     if (!m_block)
1560         return;
1561     
1562     ASSERT(m_block->isReachable);
1563     
1564     m_jit.blockHeads()[m_block->index] = m_jit.label();
1565
1566     if (!m_block->intersectionOfCFAHasVisited) {
1567         // Don't generate code for basic blocks that are unreachable according to CFA.
1568         // But to be sure that nobody has generated a jump to this block, drop in a
1569         // breakpoint here.
1570         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1571         return;
1572     }
1573
1574     m_stream->appendAndLog(VariableEvent::reset());
1575     
1576     m_jit.jitAssertHasValidCallFrame();
1577     m_jit.jitAssertTagsInPlace();
1578     m_jit.jitAssertArgumentCountSane();
1579
1580     m_state.reset();
1581     m_state.beginBasicBlock(m_block);
1582     
1583     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1584         int operand = m_block->variablesAtHead.operandForIndex(i);
1585         Node* node = m_block->variablesAtHead[i];
1586         if (!node)
1587             continue; // No need to record dead SetLocal's.
1588         
1589         VariableAccessData* variable = node->variableAccessData();
1590         DataFormat format;
1591         if (!node->refCount())
1592             continue; // No need to record dead SetLocal's.
1593         format = dataFormatFor(variable->flushFormat());
1594         m_stream->appendAndLog(
1595             VariableEvent::setLocal(
1596                 VirtualRegister(operand),
1597                 variable->machineLocal(),
1598                 format));
1599     }
1600
1601     m_origin = NodeOrigin();
1602     
1603     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1604         m_currentNode = m_block->at(m_indexInBlock);
1605         
1606         // We may have hit a contradiction that the CFA was aware of but that the JIT
1607         // didn't cause directly.
1608         if (!m_state.isValid()) {
1609             bail(DFGBailedAtTopOfBlock);
1610             return;
1611         }
1612
1613         m_interpreter.startExecuting();
1614         m_jit.setForNode(m_currentNode);
1615         m_origin = m_currentNode->origin;
1616         if (validationEnabled())
1617             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1618         m_lastGeneratedNode = m_currentNode->op();
1619         
1620         ASSERT(m_currentNode->shouldGenerate());
1621         
1622         if (verboseCompilationEnabled()) {
1623             dataLogF(
1624                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1625                 (int)m_currentNode->index(),
1626                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1627             dataLog("\n");
1628         }
1629
1630         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1631             m_jit.jitReleaseAssertNoException();
1632
1633         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.label(), m_origin.semantic);
1634
1635         compile(m_currentNode);
1636         
1637         if (belongsInMinifiedGraph(m_currentNode->op()))
1638             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1639         
1640 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1641         m_jit.clearRegisterAllocationOffsets();
1642 #endif
1643         
1644         if (!m_compileOkay) {
1645             bail(DFGBailedAtEndOfNode);
1646             return;
1647         }
1648         
1649         // Make sure that the abstract state is rematerialized for the next node.
1650         m_interpreter.executeEffects(m_indexInBlock);
1651     }
1652     
1653     // Perform the most basic verification that children have been used correctly.
1654     if (!ASSERT_DISABLED) {
1655         for (unsigned index = 0; index < m_generationInfo.size(); ++index) {
1656             GenerationInfo& info = m_generationInfo[index];
1657             RELEASE_ASSERT(!info.alive());
1658         }
1659     }
1660 }
1661
1662 // If we are making type predictions about our arguments then
1663 // we need to check that they are correct on function entry.
1664 void SpeculativeJIT::checkArgumentTypes()
1665 {
1666     ASSERT(!m_currentNode);
1667     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1668
1669     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1670         Node* node = m_jit.graph().m_arguments[i];
1671         if (!node) {
1672             // The argument is dead. We don't do any checks for such arguments.
1673             continue;
1674         }
1675         
1676         ASSERT(node->op() == SetArgument);
1677         ASSERT(node->shouldGenerate());
1678
1679         VariableAccessData* variableAccessData = node->variableAccessData();
1680         FlushFormat format = variableAccessData->flushFormat();
1681         
1682         if (format == FlushedJSValue)
1683             continue;
1684         
1685         VirtualRegister virtualRegister = variableAccessData->local();
1686
1687         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1688         
1689 #if USE(JSVALUE64)
1690         switch (format) {
1691         case FlushedInt32: {
1692             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1693             break;
1694         }
1695         case FlushedBoolean: {
1696             GPRTemporary temp(this);
1697             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1698             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1699             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1700             break;
1701         }
1702         case FlushedCell: {
1703             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1704             break;
1705         }
1706         default:
1707             RELEASE_ASSERT_NOT_REACHED();
1708             break;
1709         }
1710 #else
1711         switch (format) {
1712         case FlushedInt32: {
1713             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1714             break;
1715         }
1716         case FlushedBoolean: {
1717             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1718             break;
1719         }
1720         case FlushedCell: {
1721             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1722             break;
1723         }
1724         default:
1725             RELEASE_ASSERT_NOT_REACHED();
1726             break;
1727         }
1728 #endif
1729     }
1730
1731     m_origin = NodeOrigin();
1732 }
1733
1734 bool SpeculativeJIT::compile()
1735 {
1736     checkArgumentTypes();
1737     
1738     ASSERT(!m_currentNode);
1739     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1740         m_jit.setForBlockIndex(blockIndex);
1741         m_block = m_jit.graph().block(blockIndex);
1742         compileCurrentBlock();
1743     }
1744     linkBranches();
1745     return true;
1746 }
1747
1748 void SpeculativeJIT::createOSREntries()
1749 {
1750     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1751         BasicBlock* block = m_jit.graph().block(blockIndex);
1752         if (!block)
1753             continue;
1754         if (!block->isOSRTarget)
1755             continue;
1756         
1757         // Currently we don't have OSR entry trampolines. We could add them
1758         // here if need be.
1759         m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1760     }
1761 }
1762
1763 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1764 {
1765     unsigned osrEntryIndex = 0;
1766     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1767         BasicBlock* block = m_jit.graph().block(blockIndex);
1768         if (!block)
1769             continue;
1770         if (!block->isOSRTarget)
1771             continue;
1772         m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
1773     }
1774     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
1775     
1776     if (verboseCompilationEnabled()) {
1777         DumpContext dumpContext;
1778         dataLog("OSR Entries:\n");
1779         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
1780             dataLog("    ", inContext(entryData, &dumpContext), "\n");
1781         if (!dumpContext.isEmpty())
1782             dumpContext.dump(WTF::dataFile());
1783     }
1784 }
1785
1786 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
1787 {
1788     Edge child3 = m_jit.graph().varArgChild(node, 2);
1789     Edge child4 = m_jit.graph().varArgChild(node, 3);
1790
1791     ArrayMode arrayMode = node->arrayMode();
1792     
1793     GPRReg baseReg = base.gpr();
1794     GPRReg propertyReg = property.gpr();
1795     
1796     SpeculateDoubleOperand value(this, child3);
1797
1798     FPRReg valueReg = value.fpr();
1799     
1800     DFG_TYPE_CHECK(
1801         JSValueRegs(), child3, SpecFullRealNumber,
1802         m_jit.branchDouble(
1803             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
1804     
1805     if (!m_compileOkay)
1806         return;
1807     
1808     StorageOperand storage(this, child4);
1809     GPRReg storageReg = storage.gpr();
1810
1811     if (node->op() == PutByValAlias) {
1812         // Store the value to the array.
1813         GPRReg propertyReg = property.gpr();
1814         FPRReg valueReg = value.fpr();
1815         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
1816         
1817         noResult(m_currentNode);
1818         return;
1819     }
1820     
1821     GPRTemporary temporary;
1822     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
1823
1824     MacroAssembler::Jump slowCase;
1825     
1826     if (arrayMode.isInBounds()) {
1827         speculationCheck(
1828             OutOfBounds, JSValueRegs(), 0,
1829             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
1830     } else {
1831         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1832         
1833         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
1834         
1835         if (!arrayMode.isOutOfBounds())
1836             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
1837         
1838         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
1839         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1840         
1841         inBounds.link(&m_jit);
1842     }
1843     
1844     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
1845
1846     base.use();
1847     property.use();
1848     value.use();
1849     storage.use();
1850     
1851     if (arrayMode.isOutOfBounds()) {
1852         addSlowPathGenerator(
1853             slowPathCall(
1854                 slowCase, this,
1855                 m_jit.codeBlock()->isStrictMode() ? operationPutDoubleByValBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1856                 NoResult, baseReg, propertyReg, valueReg));
1857     }
1858
1859     noResult(m_currentNode, UseChildrenCalledExplicitly);
1860 }
1861
1862 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
1863 {
1864     SpeculateCellOperand string(this, node->child1());
1865     SpeculateStrictInt32Operand index(this, node->child2());
1866     StorageOperand storage(this, node->child3());
1867
1868     GPRReg stringReg = string.gpr();
1869     GPRReg indexReg = index.gpr();
1870     GPRReg storageReg = storage.gpr();
1871     
1872     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
1873
1874     // unsigned comparison so we can filter out negative indices and indices that are too large
1875     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
1876
1877     GPRTemporary scratch(this);
1878     GPRReg scratchReg = scratch.gpr();
1879
1880     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
1881
1882     // Load the character into scratchReg
1883     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1884
1885     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
1886     JITCompiler::Jump cont8Bit = m_jit.jump();
1887
1888     is16Bit.link(&m_jit);
1889
1890     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
1891
1892     cont8Bit.link(&m_jit);
1893
1894     int32Result(scratchReg, m_currentNode);
1895 }
1896
1897 void SpeculativeJIT::compileGetByValOnString(Node* node)
1898 {
1899     SpeculateCellOperand base(this, node->child1());
1900     SpeculateStrictInt32Operand property(this, node->child2());
1901     StorageOperand storage(this, node->child3());
1902     GPRReg baseReg = base.gpr();
1903     GPRReg propertyReg = property.gpr();
1904     GPRReg storageReg = storage.gpr();
1905
1906     GPRTemporary scratch(this);
1907     GPRReg scratchReg = scratch.gpr();
1908 #if USE(JSVALUE32_64)
1909     GPRTemporary resultTag;
1910     GPRReg resultTagReg = InvalidGPRReg;
1911     if (node->arrayMode().isOutOfBounds()) {
1912         GPRTemporary realResultTag(this);
1913         resultTag.adopt(realResultTag);
1914         resultTagReg = resultTag.gpr();
1915     }
1916 #endif
1917
1918     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
1919
1920     // unsigned comparison so we can filter out negative indices and indices that are too large
1921     JITCompiler::Jump outOfBounds = m_jit.branch32(
1922         MacroAssembler::AboveOrEqual, propertyReg,
1923         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
1924     if (node->arrayMode().isInBounds())
1925         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
1926
1927     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
1928
1929     // Load the character into scratchReg
1930     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1931
1932     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
1933     JITCompiler::Jump cont8Bit = m_jit.jump();
1934
1935     is16Bit.link(&m_jit);
1936
1937     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
1938
1939     JITCompiler::Jump bigCharacter =
1940         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
1941
1942     // 8 bit string values don't need the isASCII check.
1943     cont8Bit.link(&m_jit);
1944
1945     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
1946     m_jit.addPtr(MacroAssembler::TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
1947     m_jit.loadPtr(scratchReg, scratchReg);
1948
1949     addSlowPathGenerator(
1950         slowPathCall(
1951             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
1952
1953     if (node->arrayMode().isOutOfBounds()) {
1954 #if USE(JSVALUE32_64)
1955         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
1956 #endif
1957
1958         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
1959         bool prototypeChainIsSane = false;
1960         if (globalObject->stringPrototypeChainIsSane()) {
1961             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
1962             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
1963             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
1964             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
1965             // indexed properties either.
1966             // https://bugs.webkit.org/show_bug.cgi?id=144668
1967             m_jit.graph().watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
1968             m_jit.graph().watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
1969             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
1970         }
1971         if (prototypeChainIsSane) {
1972             m_jit.graph().watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
1973             m_jit.graph().watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
1974             
1975 #if USE(JSVALUE64)
1976             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
1977                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
1978 #else
1979             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
1980                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
1981                 baseReg, propertyReg));
1982 #endif
1983         } else {
1984 #if USE(JSVALUE64)
1985             addSlowPathGenerator(
1986                 slowPathCall(
1987                     outOfBounds, this, operationGetByValStringInt,
1988                     scratchReg, baseReg, propertyReg));
1989 #else
1990             addSlowPathGenerator(
1991                 slowPathCall(
1992                     outOfBounds, this, operationGetByValStringInt,
1993                     resultTagReg, scratchReg, baseReg, propertyReg));
1994 #endif
1995         }
1996         
1997 #if USE(JSVALUE64)
1998         jsValueResult(scratchReg, m_currentNode);
1999 #else
2000         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2001 #endif
2002     } else
2003         cellResult(scratchReg, m_currentNode);
2004 }
2005
2006 void SpeculativeJIT::compileFromCharCode(Node* node)
2007 {
2008     Edge& child = node->child1();
2009     if (child.useKind() == UntypedUse) {
2010         JSValueOperand opr(this, child);
2011         JSValueRegs oprRegs = opr.jsValueRegs();
2012 #if USE(JSVALUE64)
2013         GPRTemporary result(this);
2014         JSValueRegs resultRegs = JSValueRegs(result.gpr());
2015 #else
2016         GPRTemporary resultTag(this);
2017         GPRTemporary resultPayload(this);
2018         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
2019 #endif
2020         flushRegisters();
2021         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2022         m_jit.exceptionCheck();
2023         
2024         jsValueResult(resultRegs, node);
2025         return;
2026     }
2027
2028     SpeculateStrictInt32Operand property(this, child);
2029     GPRReg propertyReg = property.gpr();
2030     GPRTemporary smallStrings(this);
2031     GPRTemporary scratch(this);
2032     GPRReg scratchReg = scratch.gpr();
2033     GPRReg smallStringsReg = smallStrings.gpr();
2034
2035     JITCompiler::JumpList slowCases;
2036     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2037     m_jit.move(MacroAssembler::TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2038     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2039
2040     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2041     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2042     cellResult(scratchReg, m_currentNode);
2043 }
2044
2045 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2046 {
2047     VirtualRegister virtualRegister = node->virtualRegister();
2048     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2049
2050     switch (info.registerFormat()) {
2051     case DataFormatStorage:
2052         RELEASE_ASSERT_NOT_REACHED();
2053
2054     case DataFormatBoolean:
2055     case DataFormatCell:
2056         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2057         return GeneratedOperandTypeUnknown;
2058
2059     case DataFormatNone:
2060     case DataFormatJSCell:
2061     case DataFormatJS:
2062     case DataFormatJSBoolean:
2063     case DataFormatJSDouble:
2064         return GeneratedOperandJSValue;
2065
2066     case DataFormatJSInt32:
2067     case DataFormatInt32:
2068         return GeneratedOperandInteger;
2069
2070     default:
2071         RELEASE_ASSERT_NOT_REACHED();
2072         return GeneratedOperandTypeUnknown;
2073     }
2074 }
2075
2076 void SpeculativeJIT::compileValueToInt32(Node* node)
2077 {
2078     switch (node->child1().useKind()) {
2079 #if USE(JSVALUE64)
2080     case Int52RepUse: {
2081         SpeculateStrictInt52Operand op1(this, node->child1());
2082         GPRTemporary result(this, Reuse, op1);
2083         GPRReg op1GPR = op1.gpr();
2084         GPRReg resultGPR = result.gpr();
2085         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2086         int32Result(resultGPR, node, DataFormatInt32);
2087         return;
2088     }
2089 #endif // USE(JSVALUE64)
2090         
2091     case DoubleRepUse: {
2092         GPRTemporary result(this);
2093         SpeculateDoubleOperand op1(this, node->child1());
2094         FPRReg fpr = op1.fpr();
2095         GPRReg gpr = result.gpr();
2096         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2097         
2098         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this, toInt32, gpr, fpr, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded));
2099         
2100         int32Result(gpr, node);
2101         return;
2102     }
2103     
2104     case NumberUse:
2105     case NotCellUse: {
2106         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2107         case GeneratedOperandInteger: {
2108             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2109             GPRTemporary result(this, Reuse, op1);
2110             m_jit.move(op1.gpr(), result.gpr());
2111             int32Result(result.gpr(), node, op1.format());
2112             return;
2113         }
2114         case GeneratedOperandJSValue: {
2115             GPRTemporary result(this);
2116 #if USE(JSVALUE64)
2117             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2118
2119             GPRReg gpr = op1.gpr();
2120             GPRReg resultGpr = result.gpr();
2121             FPRTemporary tempFpr(this);
2122             FPRReg fpr = tempFpr.fpr();
2123
2124             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2125             JITCompiler::JumpList converted;
2126
2127             if (node->child1().useKind() == NumberUse) {
2128                 DFG_TYPE_CHECK(
2129                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2130                     m_jit.branchTest64(
2131                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2132             } else {
2133                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2134                 
2135                 DFG_TYPE_CHECK(
2136                     JSValueRegs(gpr), node->child1(), ~SpecCell, m_jit.branchIfCell(JSValueRegs(gpr)));
2137                 
2138                 // It's not a cell: so true turns into 1 and all else turns into 0.
2139                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2140                 converted.append(m_jit.jump());
2141                 
2142                 isNumber.link(&m_jit);
2143             }
2144
2145             // First, if we get here we have a double encoded as a JSValue
2146             unboxDouble(gpr, resultGpr, fpr);
2147
2148             silentSpillAllRegisters(resultGpr);
2149             callOperation(toInt32, resultGpr, fpr);
2150             silentFillAllRegisters(resultGpr);
2151
2152             converted.append(m_jit.jump());
2153
2154             isInteger.link(&m_jit);
2155             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2156
2157             converted.link(&m_jit);
2158 #else
2159             Node* childNode = node->child1().node();
2160             VirtualRegister virtualRegister = childNode->virtualRegister();
2161             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2162
2163             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2164
2165             GPRReg payloadGPR = op1.payloadGPR();
2166             GPRReg resultGpr = result.gpr();
2167         
2168             JITCompiler::JumpList converted;
2169
2170             if (info.registerFormat() == DataFormatJSInt32)
2171                 m_jit.move(payloadGPR, resultGpr);
2172             else {
2173                 GPRReg tagGPR = op1.tagGPR();
2174                 FPRTemporary tempFpr(this);
2175                 FPRReg fpr = tempFpr.fpr();
2176                 FPRTemporary scratch(this);
2177
2178                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2179
2180                 if (node->child1().useKind() == NumberUse) {
2181                     DFG_TYPE_CHECK(
2182                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2183                         m_jit.branch32(
2184                             MacroAssembler::AboveOrEqual, tagGPR,
2185                             TrustedImm32(JSValue::LowestTag)));
2186                 } else {
2187                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2188                     
2189                     DFG_TYPE_CHECK(
2190                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2191                         m_jit.branchIfCell(op1.jsValueRegs()));
2192                     
2193                     // It's not a cell: so true turns into 1 and all else turns into 0.
2194                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2195                     m_jit.move(TrustedImm32(0), resultGpr);
2196                     converted.append(m_jit.jump());
2197                     
2198                     isBoolean.link(&m_jit);
2199                     m_jit.move(payloadGPR, resultGpr);
2200                     converted.append(m_jit.jump());
2201                     
2202                     isNumber.link(&m_jit);
2203                 }
2204
2205                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2206
2207                 silentSpillAllRegisters(resultGpr);
2208                 callOperation(toInt32, resultGpr, fpr);
2209                 silentFillAllRegisters(resultGpr);
2210
2211                 converted.append(m_jit.jump());
2212
2213                 isInteger.link(&m_jit);
2214                 m_jit.move(payloadGPR, resultGpr);
2215
2216                 converted.link(&m_jit);
2217             }
2218 #endif
2219             int32Result(resultGpr, node);
2220             return;
2221         }
2222         case GeneratedOperandTypeUnknown:
2223             RELEASE_ASSERT(!m_compileOkay);
2224             return;
2225         }
2226         RELEASE_ASSERT_NOT_REACHED();
2227         return;
2228     }
2229     
2230     default:
2231         ASSERT(!m_compileOkay);
2232         return;
2233     }
2234 }
2235
2236 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2237 {
2238     if (doesOverflow(node->arithMode())) {
2239         if (enableInt52()) {
2240             SpeculateInt32Operand op1(this, node->child1());
2241             GPRTemporary result(this, Reuse, op1);
2242             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2243             strictInt52Result(result.gpr(), node);
2244             return;
2245         }
2246         SpeculateInt32Operand op1(this, node->child1());
2247         FPRTemporary result(this);
2248             
2249         GPRReg inputGPR = op1.gpr();
2250         FPRReg outputFPR = result.fpr();
2251             
2252         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2253             
2254         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2255         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2256         positive.link(&m_jit);
2257             
2258         doubleResult(outputFPR, node);
2259         return;
2260     }
2261     
2262     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2263
2264     SpeculateInt32Operand op1(this, node->child1());
2265     GPRTemporary result(this);
2266
2267     m_jit.move(op1.gpr(), result.gpr());
2268
2269     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2270
2271     int32Result(result.gpr(), node, op1.format());
2272 }
2273
2274 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2275 {
2276     SpeculateDoubleOperand op1(this, node->child1());
2277     FPRTemporary scratch(this);
2278     GPRTemporary result(this);
2279     
2280     FPRReg valueFPR = op1.fpr();
2281     FPRReg scratchFPR = scratch.fpr();
2282     GPRReg resultGPR = result.gpr();
2283
2284     JITCompiler::JumpList failureCases;
2285     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2286     m_jit.branchConvertDoubleToInt32(
2287         valueFPR, resultGPR, failureCases, scratchFPR,
2288         shouldCheckNegativeZero(node->arithMode()));
2289     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2290
2291     int32Result(resultGPR, node);
2292 }
2293
2294 void SpeculativeJIT::compileDoubleRep(Node* node)
2295 {
2296     switch (node->child1().useKind()) {
2297     case RealNumberUse: {
2298         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2299         FPRTemporary result(this);
2300         
2301         JSValueRegs op1Regs = op1.jsValueRegs();
2302         FPRReg resultFPR = result.fpr();
2303         
2304 #if USE(JSVALUE64)
2305         GPRTemporary temp(this);
2306         GPRReg tempGPR = temp.gpr();
2307         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2308 #else
2309         FPRTemporary temp(this);
2310         FPRReg tempFPR = temp.fpr();
2311         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2312 #endif
2313         
2314         JITCompiler::Jump done = m_jit.branchDouble(
2315             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2316         
2317         DFG_TYPE_CHECK(
2318             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2319         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2320         
2321         done.link(&m_jit);
2322         
2323         doubleResult(resultFPR, node);
2324         return;
2325     }
2326     
2327     case NotCellUse:
2328     case NumberUse: {
2329         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2330
2331         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2332         if (isInt32Speculation(possibleTypes)) {
2333             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2334             FPRTemporary result(this);
2335             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2336             doubleResult(result.fpr(), node);
2337             return;
2338         }
2339
2340         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2341         FPRTemporary result(this);
2342
2343 #if USE(JSVALUE64)
2344         GPRTemporary temp(this);
2345
2346         GPRReg op1GPR = op1.gpr();
2347         GPRReg tempGPR = temp.gpr();
2348         FPRReg resultFPR = result.fpr();
2349         JITCompiler::JumpList done;
2350
2351         JITCompiler::Jump isInteger = m_jit.branch64(
2352             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2353
2354         if (node->child1().useKind() == NotCellUse) {
2355             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2356             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2357
2358             static const double zero = 0;
2359             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&zero), resultFPR);
2360
2361             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2362             done.append(isNull);
2363
2364             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCell,
2365                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2366
2367             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2368             static const double one = 1;
2369             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&one), resultFPR);
2370             done.append(m_jit.jump());
2371             done.append(isFalse);
2372
2373             isUndefined.link(&m_jit);
2374             static const double NaN = PNaN;
2375             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&NaN), resultFPR);
2376             done.append(m_jit.jump());
2377
2378             isNumber.link(&m_jit);
2379         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2380             typeCheck(
2381                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2382                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2383         }
2384
2385         unboxDouble(op1GPR, tempGPR, resultFPR);
2386         done.append(m_jit.jump());
2387     
2388         isInteger.link(&m_jit);
2389         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2390         done.link(&m_jit);
2391 #else // USE(JSVALUE64) -> this is the 32_64 case
2392         FPRTemporary temp(this);
2393     
2394         GPRReg op1TagGPR = op1.tagGPR();
2395         GPRReg op1PayloadGPR = op1.payloadGPR();
2396         FPRReg tempFPR = temp.fpr();
2397         FPRReg resultFPR = result.fpr();
2398         JITCompiler::JumpList done;
2399     
2400         JITCompiler::Jump isInteger = m_jit.branch32(
2401             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2402
2403         if (node->child1().useKind() == NotCellUse) {
2404             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2405             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2406
2407             static const double zero = 0;
2408             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&zero), resultFPR);
2409
2410             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2411             done.append(isNull);
2412
2413             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2414
2415             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2416             static const double one = 1;
2417             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&one), resultFPR);
2418             done.append(m_jit.jump());
2419             done.append(isFalse);
2420
2421             isUndefined.link(&m_jit);
2422             static const double NaN = PNaN;
2423             m_jit.loadDouble(MacroAssembler::TrustedImmPtr(&NaN), resultFPR);
2424             done.append(m_jit.jump());
2425
2426             isNumber.link(&m_jit);
2427         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2428             typeCheck(
2429                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2430                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2431         }
2432
2433         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2434         done.append(m_jit.jump());
2435     
2436         isInteger.link(&m_jit);
2437         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2438         done.link(&m_jit);
2439 #endif // USE(JSVALUE64)
2440     
2441         doubleResult(resultFPR, node);
2442         return;
2443     }
2444         
2445 #if USE(JSVALUE64)
2446     case Int52RepUse: {
2447         SpeculateStrictInt52Operand value(this, node->child1());
2448         FPRTemporary result(this);
2449         
2450         GPRReg valueGPR = value.gpr();
2451         FPRReg resultFPR = result.fpr();
2452
2453         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2454         
2455         doubleResult(resultFPR, node);
2456         return;
2457     }
2458 #endif // USE(JSVALUE64)
2459         
2460     default:
2461         RELEASE_ASSERT_NOT_REACHED();
2462         return;
2463     }
2464 }
2465
2466 void SpeculativeJIT::compileValueRep(Node* node)
2467 {
2468     switch (node->child1().useKind()) {
2469     case DoubleRepUse: {
2470         SpeculateDoubleOperand value(this, node->child1());
2471         JSValueRegsTemporary result(this);
2472         
2473         FPRReg valueFPR = value.fpr();
2474         JSValueRegs resultRegs = result.regs();
2475         
2476         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2477         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2478         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2479         // local was purified.
2480         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2481             m_jit.purifyNaN(valueFPR);
2482
2483         boxDouble(valueFPR, resultRegs);
2484         
2485         jsValueResult(resultRegs, node);
2486         return;
2487     }
2488         
2489 #if USE(JSVALUE64)
2490     case Int52RepUse: {
2491         SpeculateStrictInt52Operand value(this, node->child1());
2492         GPRTemporary result(this);
2493         
2494         GPRReg valueGPR = value.gpr();
2495         GPRReg resultGPR = result.gpr();
2496         
2497         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2498         
2499         jsValueResult(resultGPR, node);
2500         return;
2501     }
2502 #endif // USE(JSVALUE64)
2503         
2504     default:
2505         RELEASE_ASSERT_NOT_REACHED();
2506         return;
2507     }
2508 }
2509
2510 static double clampDoubleToByte(double d)
2511 {
2512     d += 0.5;
2513     if (!(d > 0))
2514         d = 0;
2515     else if (d > 255)
2516         d = 255;
2517     return d;
2518 }
2519
2520 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2521 {
2522     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2523     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2524     jit.xorPtr(result, result);
2525     MacroAssembler::Jump clamped = jit.jump();
2526     tooBig.link(&jit);
2527     jit.move(JITCompiler::TrustedImm32(255), result);
2528     clamped.link(&jit);
2529     inBounds.link(&jit);
2530 }
2531
2532 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2533 {
2534     // Unordered compare so we pick up NaN
2535     static const double zero = 0;
2536     static const double byteMax = 255;
2537     static const double half = 0.5;
2538     jit.loadDouble(MacroAssembler::TrustedImmPtr(&zero), scratch);
2539     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2540     jit.loadDouble(MacroAssembler::TrustedImmPtr(&byteMax), scratch);
2541     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2542     
2543     jit.loadDouble(MacroAssembler::TrustedImmPtr(&half), scratch);
2544     // FIXME: This should probably just use a floating point round!
2545     // https://bugs.webkit.org/show_bug.cgi?id=72054
2546     jit.addDouble(source, scratch);
2547     jit.truncateDoubleToInt32(scratch, result);   
2548     MacroAssembler::Jump truncatedInt = jit.jump();
2549     
2550     tooSmall.link(&jit);
2551     jit.xorPtr(result, result);
2552     MacroAssembler::Jump zeroed = jit.jump();
2553     
2554     tooBig.link(&jit);
2555     jit.move(JITCompiler::TrustedImm32(255), result);
2556     
2557     truncatedInt.link(&jit);
2558     zeroed.link(&jit);
2559
2560 }
2561
2562 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2563 {
2564     if (node->op() == PutByValAlias)
2565         return JITCompiler::Jump();
2566     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2567         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2568     if (view) {
2569         uint32_t length = view->length();
2570         Node* indexNode = m_jit.graph().child(node, 1).node();
2571         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2572             return JITCompiler::Jump();
2573         return m_jit.branch32(
2574             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2575     }
2576     return m_jit.branch32(
2577         MacroAssembler::AboveOrEqual, indexGPR,
2578         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2579 }
2580
2581 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2582 {
2583     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2584     if (!jump.isSet())
2585         return;
2586     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2587 }
2588
2589 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2590 {
2591     ASSERT(isInt(type));
2592     
2593     SpeculateCellOperand base(this, node->child1());
2594     SpeculateStrictInt32Operand property(this, node->child2());
2595     StorageOperand storage(this, node->child3());
2596
2597     GPRReg baseReg = base.gpr();
2598     GPRReg propertyReg = property.gpr();
2599     GPRReg storageReg = storage.gpr();
2600
2601     GPRTemporary result(this);
2602     GPRReg resultReg = result.gpr();
2603
2604     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2605
2606     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2607     switch (elementSize(type)) {
2608     case 1:
2609         if (isSigned(type))
2610             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2611         else
2612             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2613         break;
2614     case 2:
2615         if (isSigned(type))
2616             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2617         else
2618             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2619         break;
2620     case 4:
2621         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2622         break;
2623     default:
2624         CRASH();
2625     }
2626     if (elementSize(type) < 4 || isSigned(type)) {
2627         int32Result(resultReg, node);
2628         return;
2629     }
2630     
2631     ASSERT(elementSize(type) == 4 && !isSigned(type));
2632     if (node->shouldSpeculateInt32()) {
2633         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2634         int32Result(resultReg, node);
2635         return;
2636     }
2637     
2638 #if USE(JSVALUE64)
2639     if (node->shouldSpeculateAnyInt()) {
2640         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2641         strictInt52Result(resultReg, node);
2642         return;
2643     }
2644 #endif
2645     
2646     FPRTemporary fresult(this);
2647     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2648     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2649     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2650     positive.link(&m_jit);
2651     doubleResult(fresult.fpr(), node);
2652 }
2653
2654 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
2655 {
2656     ASSERT(isInt(type));
2657     
2658     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
2659     GPRReg storageReg = storage.gpr();
2660     
2661     Edge valueUse = m_jit.graph().varArgChild(node, 2);
2662     
2663     GPRTemporary value;
2664     GPRReg valueGPR = InvalidGPRReg;
2665     
2666     if (valueUse->isConstant()) {
2667         JSValue jsValue = valueUse->asJSValue();
2668         if (!jsValue.isNumber()) {
2669             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2670             noResult(node);
2671             return;
2672         }
2673         double d = jsValue.asNumber();
2674         if (isClamped(type)) {
2675             ASSERT(elementSize(type) == 1);
2676             d = clampDoubleToByte(d);
2677         }
2678         GPRTemporary scratch(this);
2679         GPRReg scratchReg = scratch.gpr();
2680         m_jit.move(Imm32(toInt32(d)), scratchReg);
2681         value.adopt(scratch);
2682         valueGPR = scratchReg;
2683     } else {
2684         switch (valueUse.useKind()) {
2685         case Int32Use: {
2686             SpeculateInt32Operand valueOp(this, valueUse);
2687             GPRTemporary scratch(this);
2688             GPRReg scratchReg = scratch.gpr();
2689             m_jit.move(valueOp.gpr(), scratchReg);
2690             if (isClamped(type)) {
2691                 ASSERT(elementSize(type) == 1);
2692                 compileClampIntegerToByte(m_jit, scratchReg);
2693             }
2694             value.adopt(scratch);
2695             valueGPR = scratchReg;
2696             break;
2697         }
2698             
2699 #if USE(JSVALUE64)
2700         case Int52RepUse: {
2701             SpeculateStrictInt52Operand valueOp(this, valueUse);
2702             GPRTemporary scratch(this);
2703             GPRReg scratchReg = scratch.gpr();
2704             m_jit.move(valueOp.gpr(), scratchReg);
2705             if (isClamped(type)) {
2706                 ASSERT(elementSize(type) == 1);
2707                 MacroAssembler::Jump inBounds = m_jit.branch64(
2708                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2709                 MacroAssembler::Jump tooBig = m_jit.branch64(
2710                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2711                 m_jit.move(TrustedImm32(0), scratchReg);
2712                 MacroAssembler::Jump clamped = m_jit.jump();
2713                 tooBig.link(&m_jit);
2714                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2715                 clamped.link(&m_jit);
2716                 inBounds.link(&m_jit);
2717             }
2718             value.adopt(scratch);
2719             valueGPR = scratchReg;
2720             break;
2721         }
2722 #endif // USE(JSVALUE64)
2723             
2724         case DoubleRepUse: {
2725             if (isClamped(type)) {
2726                 ASSERT(elementSize(type) == 1);
2727                 SpeculateDoubleOperand valueOp(this, valueUse);
2728                 GPRTemporary result(this);
2729                 FPRTemporary floatScratch(this);
2730                 FPRReg fpr = valueOp.fpr();
2731                 GPRReg gpr = result.gpr();
2732                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
2733                 value.adopt(result);
2734                 valueGPR = gpr;
2735             } else {
2736                 SpeculateDoubleOperand valueOp(this, valueUse);
2737                 GPRTemporary result(this);
2738                 FPRReg fpr = valueOp.fpr();
2739                 GPRReg gpr = result.gpr();
2740                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
2741                 m_jit.xorPtr(gpr, gpr);
2742                 MacroAssembler::Jump fixed = m_jit.jump();
2743                 notNaN.link(&m_jit);
2744                 
2745                 MacroAssembler::Jump failed = m_jit.branchTruncateDoubleToInt32(
2746                     fpr, gpr, MacroAssembler::BranchIfTruncateFailed);
2747                 
2748                 addSlowPathGenerator(slowPathCall(failed, this, toInt32, gpr, fpr, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded));
2749                 
2750                 fixed.link(&m_jit);
2751                 value.adopt(result);
2752                 valueGPR = gpr;
2753             }
2754             break;
2755         }
2756             
2757         default:
2758             RELEASE_ASSERT_NOT_REACHED();
2759             break;
2760         }
2761     }
2762     
2763     ASSERT_UNUSED(valueGPR, valueGPR != property);
2764     ASSERT(valueGPR != base);
2765     ASSERT(valueGPR != storageReg);
2766     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
2767     if (node->arrayMode().isInBounds() && outOfBounds.isSet()) {
2768         speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2769         outOfBounds = MacroAssembler::Jump();
2770     }
2771
2772     switch (elementSize(type)) {
2773     case 1:
2774         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
2775         break;
2776     case 2:
2777         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
2778         break;
2779     case 4:
2780         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
2781         break;
2782     default:
2783         CRASH();
2784     }
2785     if (outOfBounds.isSet())
2786         outOfBounds.link(&m_jit);
2787     noResult(node);
2788 }
2789
2790 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
2791 {
2792     ASSERT(isFloat(type));
2793     
2794     SpeculateCellOperand base(this, node->child1());
2795     SpeculateStrictInt32Operand property(this, node->child2());
2796     StorageOperand storage(this, node->child3());
2797
2798     GPRReg baseReg = base.gpr();
2799     GPRReg propertyReg = property.gpr();
2800     GPRReg storageReg = storage.gpr();
2801
2802     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2803
2804     FPRTemporary result(this);
2805     FPRReg resultReg = result.fpr();
2806     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2807     switch (elementSize(type)) {
2808     case 4:
2809         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2810         m_jit.convertFloatToDouble(resultReg, resultReg);
2811         break;
2812     case 8: {
2813         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
2814         break;
2815     }
2816     default:
2817         RELEASE_ASSERT_NOT_REACHED();
2818     }
2819     
2820     doubleResult(resultReg, node);
2821 }
2822
2823 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
2824 {
2825     ASSERT(isFloat(type));
2826     
2827     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
2828     GPRReg storageReg = storage.gpr();
2829     
2830     Edge baseUse = m_jit.graph().varArgChild(node, 0);
2831     Edge valueUse = m_jit.graph().varArgChild(node, 2);
2832
2833     SpeculateDoubleOperand valueOp(this, valueUse);
2834     FPRTemporary scratch(this);
2835     FPRReg valueFPR = valueOp.fpr();
2836     FPRReg scratchFPR = scratch.fpr();
2837
2838     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
2839     
2840     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
2841     if (node->arrayMode().isInBounds() && outOfBounds.isSet()) {
2842         speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2843         outOfBounds = MacroAssembler::Jump();
2844     }
2845     
2846     switch (elementSize(type)) {
2847     case 4: {
2848         m_jit.moveDouble(valueFPR, scratchFPR);
2849         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
2850         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
2851         break;
2852     }
2853     case 8:
2854         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
2855         break;
2856     default:
2857         RELEASE_ASSERT_NOT_REACHED();
2858     }
2859     if (outOfBounds.isSet())
2860         outOfBounds.link(&m_jit);
2861     noResult(node);
2862 }
2863
2864 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg)
2865 {
2866     // Check that prototype is an object.
2867     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
2868     
2869     // Initialize scratchReg with the value being checked.
2870     m_jit.move(valueReg, scratchReg);
2871     
2872     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
2873     MacroAssembler::Label loop(&m_jit);
2874     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
2875         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
2876     m_jit.emitLoadStructure(scratchReg, scratchReg, scratch2Reg);
2877     m_jit.loadPtr(MacroAssembler::Address(scratchReg, Structure::prototypeOffset() + CellPayloadOffset), scratchReg);
2878     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
2879 #if USE(JSVALUE64)
2880     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
2881 #else
2882     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
2883 #endif
2884     
2885     // No match - result is false.
2886 #if USE(JSVALUE64)
2887     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
2888 #else
2889     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
2890 #endif
2891     MacroAssembler::JumpList doneJumps; 
2892     doneJumps.append(m_jit.jump());
2893
2894     performDefaultHasInstance.link(&m_jit);
2895     silentSpillAllRegisters(scratchReg);
2896     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg); 
2897     silentFillAllRegisters(scratchReg);
2898     m_jit.exceptionCheck();
2899 #if USE(JSVALUE64)
2900     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
2901 #endif
2902     doneJumps.append(m_jit.jump());
2903     
2904     isInstance.link(&m_jit);
2905 #if USE(JSVALUE64)
2906     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), scratchReg);
2907 #else
2908     m_jit.move(MacroAssembler::TrustedImm32(1), scratchReg);
2909 #endif
2910     
2911     doneJumps.link(&m_jit);
2912 }
2913
2914 void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
2915 {
2916     SpeculateCellOperand base(this, node->child1());
2917
2918     GPRReg baseGPR = base.gpr();
2919
2920     speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
2921
2922     noResult(node);
2923 }
2924
2925 void SpeculativeJIT::compileInstanceOf(Node* node)
2926 {
2927     if (node->child1().useKind() == UntypedUse) {
2928         // It might not be a cell. Speculate less aggressively.
2929         // Or: it might only be used once (i.e. by us), so we get zero benefit
2930         // from speculating any more aggressively than we absolutely need to.
2931         
2932         JSValueOperand value(this, node->child1());
2933         SpeculateCellOperand prototype(this, node->child2());
2934         GPRTemporary scratch(this);
2935         GPRTemporary scratch2(this);
2936         
2937         GPRReg prototypeReg = prototype.gpr();
2938         GPRReg scratchReg = scratch.gpr();
2939         GPRReg scratch2Reg = scratch2.gpr();
2940         
2941         MacroAssembler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs());
2942         GPRReg valueReg = value.jsValueRegs().payloadGPR();
2943         moveFalseTo(scratchReg);
2944
2945         MacroAssembler::Jump done = m_jit.jump();
2946         
2947         isCell.link(&m_jit);
2948         
2949         compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg);
2950         
2951         done.link(&m_jit);
2952
2953         blessedBooleanResult(scratchReg, node);
2954         return;
2955     }
2956     
2957     SpeculateCellOperand value(this, node->child1());
2958     SpeculateCellOperand prototype(this, node->child2());
2959     
2960     GPRTemporary scratch(this);
2961     GPRTemporary scratch2(this);
2962     
2963     GPRReg valueReg = value.gpr();
2964     GPRReg prototypeReg = prototype.gpr();
2965     GPRReg scratchReg = scratch.gpr();
2966     GPRReg scratch2Reg = scratch2.gpr();
2967     
2968     compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg, scratch2Reg);
2969
2970     blessedBooleanResult(scratchReg, node);
2971 }
2972
2973 template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
2974 void SpeculativeJIT::emitUntypedBitOp(Node* node)
2975 {
2976     Edge& leftChild = node->child1();
2977     Edge& rightChild = node->child2();
2978
2979     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
2980         JSValueOperand left(this, leftChild);
2981         JSValueOperand right(this, rightChild);
2982         JSValueRegs leftRegs = left.jsValueRegs();
2983         JSValueRegs rightRegs = right.jsValueRegs();
2984 #if USE(JSVALUE64)
2985         GPRTemporary result(this);
2986         JSValueRegs resultRegs = JSValueRegs(result.gpr());
2987 #else
2988         GPRTemporary resultTag(this);
2989         GPRTemporary resultPayload(this);
2990         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
2991 #endif
2992         flushRegisters();
2993         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
2994         m_jit.exceptionCheck();
2995
2996         jsValueResult(resultRegs, node);
2997         return;
2998     }
2999
3000     Optional<JSValueOperand> left;
3001     Optional<JSValueOperand> right;
3002
3003     JSValueRegs leftRegs;
3004     JSValueRegs rightRegs;
3005
3006 #if USE(JSVALUE64)
3007     GPRTemporary result(this);
3008     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3009     GPRTemporary scratch(this);
3010     GPRReg scratchGPR = scratch.gpr();
3011 #else
3012     GPRTemporary resultTag(this);
3013     GPRTemporary resultPayload(this);
3014     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3015     GPRReg scratchGPR = resultTag.gpr();
3016 #endif
3017
3018     SnippetOperand leftOperand;
3019     SnippetOperand rightOperand;
3020
3021     // The snippet generator does not support both operands being constant. If the left
3022     // operand is already const, we'll ignore the right operand's constness.
3023     if (leftChild->isInt32Constant())
3024         leftOperand.setConstInt32(leftChild->asInt32());
3025     else if (rightChild->isInt32Constant())
3026         rightOperand.setConstInt32(rightChild->asInt32());
3027
3028     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3029
3030     if (!leftOperand.isConst()) {
3031         left = JSValueOperand(this, leftChild);
3032         leftRegs = left->jsValueRegs();
3033     }
3034     if (!rightOperand.isConst()) {
3035         right = JSValueOperand(this, rightChild);
3036         rightRegs = right->jsValueRegs();
3037     }
3038
3039     SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3040     gen.generateFastPath(m_jit);
3041
3042     ASSERT(gen.didEmitFastPath());
3043     gen.endJumpList().append(m_jit.jump());
3044
3045     gen.slowPathJumpList().link(&m_jit);
3046     silentSpillAllRegisters(resultRegs);
3047
3048     if (leftOperand.isConst()) {
3049         leftRegs = resultRegs;
3050         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3051     } else if (rightOperand.isConst()) {
3052         rightRegs = resultRegs;
3053         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3054     }
3055
3056     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3057
3058     silentFillAllRegisters(resultRegs);
3059     m_jit.exceptionCheck();
3060
3061     gen.endJumpList().link(&m_jit);
3062     jsValueResult(resultRegs, node);
3063 }
3064
3065 void SpeculativeJIT::compileBitwiseOp(Node* node)
3066 {
3067     NodeType op = node->op();
3068     Edge& leftChild = node->child1();
3069     Edge& rightChild = node->child2();
3070
3071     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3072         switch (op) {
3073         case BitAnd:
3074             emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3075             return;
3076         case BitOr:
3077             emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3078             return;
3079         case BitXor:
3080             emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3081             return;
3082         default:
3083             RELEASE_ASSERT_NOT_REACHED();
3084         }
3085     }
3086
3087     if (leftChild->isInt32Constant()) {
3088         SpeculateInt32Operand op2(this, rightChild);
3089         GPRTemporary result(this, Reuse, op2);
3090
3091         bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3092
3093         int32Result(result.gpr(), node);
3094
3095     } else if (rightChild->isInt32Constant()) {
3096         SpeculateInt32Operand op1(this, leftChild);
3097         GPRTemporary result(this, Reuse, op1);
3098
3099         bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3100
3101         int32Result(result.gpr(), node);
3102
3103     } else {
3104         SpeculateInt32Operand op1(this, leftChild);
3105         SpeculateInt32Operand op2(this, rightChild);
3106         GPRTemporary result(this, Reuse, op1, op2);
3107         
3108         GPRReg reg1 = op1.gpr();
3109         GPRReg reg2 = op2.gpr();
3110         bitOp(op, reg1, reg2, result.gpr());
3111         
3112         int32Result(result.gpr(), node);
3113     }
3114 }
3115
3116 void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3117 {
3118     J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3119         ? operationValueBitRShift : operationValueBitURShift;
3120     JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3121         ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3122
3123     Edge& leftChild = node->child1();
3124     Edge& rightChild = node->child2();
3125
3126     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3127         JSValueOperand left(this, leftChild);
3128         JSValueOperand right(this, rightChild);
3129         JSValueRegs leftRegs = left.jsValueRegs();
3130         JSValueRegs rightRegs = right.jsValueRegs();
3131 #if USE(JSVALUE64)
3132         GPRTemporary result(this);
3133         JSValueRegs resultRegs = JSValueRegs(result.gpr());
3134 #else
3135         GPRTemporary resultTag(this);
3136         GPRTemporary resultPayload(this);
3137         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3138 #endif
3139         flushRegisters();
3140         callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3141         m_jit.exceptionCheck();
3142
3143         jsValueResult(resultRegs, node);
3144         return;
3145     }
3146
3147     Optional<JSValueOperand> left;
3148     Optional<JSValueOperand> right;
3149
3150     JSValueRegs leftRegs;
3151     JSValueRegs rightRegs;
3152
3153     FPRTemporary leftNumber(this);
3154     FPRReg leftFPR = leftNumber.fpr();
3155
3156 #if USE(JSVALUE64)
3157     GPRTemporary result(this);
3158     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3159     GPRTemporary scratch(this);
3160     GPRReg scratchGPR = scratch.gpr();
3161     FPRReg scratchFPR = InvalidFPRReg;
3162 #else
3163     GPRTemporary resultTag(this);
3164     GPRTemporary resultPayload(this);
3165     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3166     GPRReg scratchGPR = resultTag.gpr();
3167     FPRTemporary fprScratch(this);
3168     FPRReg scratchFPR = fprScratch.fpr();
3169 #endif
3170
3171     SnippetOperand leftOperand;
3172     SnippetOperand rightOperand;
3173
3174     // The snippet generator does not support both operands being constant. If the left
3175     // operand is already const, we'll ignore the right operand's constness.
3176     if (leftChild->isInt32Constant())
3177         leftOperand.setConstInt32(leftChild->asInt32());
3178     else if (rightChild->isInt32Constant())
3179         rightOperand.setConstInt32(rightChild->asInt32());
3180
3181     RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3182
3183     if (!leftOperand.isConst()) {
3184         left = JSValueOperand(this, leftChild);
3185         leftRegs = left->jsValueRegs();
3186     }
3187     if (!rightOperand.isConst()) {
3188         right = JSValueOperand(this, rightChild);
3189         rightRegs = right->jsValueRegs();
3190     }
3191
3192     JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3193         leftFPR, scratchGPR, scratchFPR, shiftType);
3194     gen.generateFastPath(m_jit);
3195
3196     ASSERT(gen.didEmitFastPath());
3197     gen.endJumpList().append(m_jit.jump());
3198
3199     gen.slowPathJumpList().link(&m_jit);
3200     silentSpillAllRegisters(resultRegs);
3201
3202     if (leftOperand.isConst()) {
3203         leftRegs = resultRegs;
3204         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3205     } else if (rightOperand.isConst()) {
3206         rightRegs = resultRegs;
3207         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3208     }
3209
3210     callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3211
3212     silentFillAllRegisters(resultRegs);
3213     m_jit.exceptionCheck();
3214
3215     gen.endJumpList().link(&m_jit);
3216     jsValueResult(resultRegs, node);
3217     return;
3218 }
3219
3220 void SpeculativeJIT::compileShiftOp(Node* node)
3221 {
3222     NodeType op = node->op();
3223     Edge& leftChild = node->child1();
3224     Edge& rightChild = node->child2();
3225
3226     if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3227         switch (op) {
3228         case BitLShift:
3229             emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3230             return;
3231         case BitRShift:
3232         case BitURShift:
3233             emitUntypedRightShiftBitOp(node);
3234             return;
3235         default:
3236             RELEASE_ASSERT_NOT_REACHED();
3237         }
3238     }
3239
3240     if (rightChild->isInt32Constant()) {
3241         SpeculateInt32Operand op1(this, leftChild);
3242         GPRTemporary result(this, Reuse, op1);
3243
3244         shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3245
3246         int32Result(result.gpr(), node);
3247     } else {
3248         // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3249         SpeculateInt32Operand op1(this, leftChild);
3250         SpeculateInt32Operand op2(this, rightChild);
3251         GPRTemporary result(this, Reuse, op1);
3252
3253         GPRReg reg1 = op1.gpr();
3254         GPRReg reg2 = op2.gpr();
3255         shiftOp(op, reg1, reg2, result.gpr());
3256
3257         int32Result(result.gpr(), node);
3258     }
3259 }
3260
3261 void SpeculativeJIT::compileValueAdd(Node* node)
3262 {
3263     Edge& leftChild = node->child1();
3264     Edge& rightChild = node->child2();
3265
3266     if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3267         JSValueOperand left(this, leftChild);
3268         JSValueOperand right(this, rightChild);
3269         JSValueRegs leftRegs = left.jsValueRegs();
3270         JSValueRegs rightRegs = right.jsValueRegs();
3271 #if USE(JSVALUE64)
3272         GPRTemporary result(this);
3273         JSValueRegs resultRegs = JSValueRegs(result.gpr());
3274 #else
3275         GPRTemporary resultTag(this);
3276         GPRTemporary resultPayload(this);
3277         JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3278 #endif
3279         flushRegisters();
3280         callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3281         m_jit.exceptionCheck();
3282     
3283         jsValueResult(resultRegs, node);
3284         return;
3285     }
3286
3287     Optional<JSValueOperand> left;
3288     Optional<JSValueOperand> right;
3289
3290     JSValueRegs leftRegs;
3291     JSValueRegs rightRegs;
3292
3293     FPRTemporary leftNumber(this);
3294     FPRTemporary rightNumber(this);
3295     FPRReg leftFPR = leftNumber.fpr();
3296     FPRReg rightFPR = rightNumber.fpr();
3297
3298 #if USE(JSVALUE64)
3299     GPRTemporary result(this);
3300     JSValueRegs resultRegs = JSValueRegs(result.gpr());
3301     GPRTemporary scratch(this);
3302     GPRReg scratchGPR = scratch.gpr();
3303     FPRReg scratchFPR = InvalidFPRReg;
3304 #else
3305     GPRTemporary resultTag(this);
3306     GPRTemporary resultPayload(this);
3307     JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3308     GPRReg scratchGPR = resultTag.gpr();
3309     FPRTemporary fprScratch(this);
3310     FPRReg scratchFPR = fprScratch.fpr();
3311 #endif
3312
3313     SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
3314     SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
3315
3316     // The snippet generator does not support both operands being constant. If the left
3317     // operand is already const, we'll ignore the right operand's constness.
3318     if (leftChild->isInt32Constant())
3319         leftOperand.setConstInt32(leftChild->asInt32());
3320     else if (rightChild->isInt32Constant())
3321         rightOperand.setConstInt32(rightChild->asInt32());
3322
3323     ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3324
3325     if (!leftOperand.isConst()) {
3326         left = JSValueOperand(this, leftChild);
3327         leftRegs = left->jsValueRegs();
3328     }
3329     if (!rightOperand.isConst()) {
3330         right = JSValueOperand(this, rightChild);
3331         rightRegs = right->jsValueRegs();
3332     }
3333
3334     JITAddGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3335         leftFPR, rightFPR, scratchGPR, scratchFPR);
3336     gen.generateFastPath(m_jit);
3337
3338     ASSERT(gen.didEmitFastPath());
3339     gen.endJumpList().append(m_jit.jump());
3340
3341     gen.slowPathJumpList().link(&m_jit);
3342
3343     silentSpillAllRegisters(resultRegs);
3344
3345     if (leftOperand.isConst()) {
3346         leftRegs = resultRegs;
3347         m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3348     } else if (rightOperand.isConst()) {
3349         rightRegs = resultRegs;
3350         m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3351     }
3352