The DFG should be able to tier-up and OSR enter into the FTL
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToLLVM.cpp
1 /*
2  * Copyright (C) 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToLLVM.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "CodeBlockWithJITType.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGInPlaceAbstractState.h"
34 #include "FTLAbstractHeapRepository.h"
35 #include "FTLExitThunkGenerator.h"
36 #include "FTLForOSREntryJITCode.h"
37 #include "FTLFormattedValue.h"
38 #include "FTLLoweredNodeValue.h"
39 #include "FTLOutput.h"
40 #include "FTLThunks.h"
41 #include "FTLValueSource.h"
42 #include "LinkBuffer.h"
43 #include "OperandsInlines.h"
44 #include "Operations.h"
45 #include <wtf/ProcessID.h>
46
47 namespace JSC { namespace FTL {
48
49 using namespace DFG;
50
51 static int compileCounter;
52
53 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
54 // significantly less dead code.
55 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
56         FormattedValue _ftc_lowValue = (lowValue);                      \
57         Edge _ftc_highValue = (highValue);                              \
58         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
59         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
60             break;                                                      \
61         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
62     } while (false)
63
64 class LowerDFGToLLVM {
65 public:
66     LowerDFGToLLVM(State& state)
67         : m_graph(state.graph)
68         , m_ftlState(state)
69         , m_heaps(state.context)
70         , m_out(state.context)
71         , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
72         , m_lastSetOperand(std::numeric_limits<int>::max())
73         , m_exitThunkGenerator(state)
74         , m_state(state.graph)
75         , m_interpreter(state.graph, m_state)
76     {
77     }
78     
79     void lower()
80     {
81         CString name;
82         if (verboseCompilationEnabled()) {
83             name = toCString(
84                 "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
85                 "_", codeBlock()->hash());
86         } else
87             name = "jsBody";
88         
89         m_graph.m_dominators.computeIfNecessary(m_graph);
90         
91         m_ftlState.module =
92             LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
93         
94         m_ftlState.function = addFunction(
95             m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
96         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
97         
98         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
99         
100         m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
101         m_out.appendTo(m_prologue);
102         createPhiVariables();
103         
104         m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
105
106         m_callFrame = m_out.param(0);
107         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
108         m_tagMask = m_out.constInt64(TagMask);
109         
110         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
111             m_highBlock = m_graph.block(blockIndex);
112             if (!m_highBlock)
113                 continue;
114             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
115         }
116         
117         Vector<BasicBlock*> depthFirst;
118         m_graph.getBlocksInDepthFirstOrder(depthFirst);
119         for (unsigned i = 0; i < depthFirst.size(); ++i)
120             compileBlock(depthFirst[i]);
121         
122         // And now complete the initialization block.
123         linkOSRExitsAndCompleteInitializationBlocks();
124
125         if (Options::dumpLLVMIR())
126             dumpModule(m_ftlState.module);
127         
128         if (verboseCompilationEnabled())
129             m_ftlState.dumpState("after lowering");
130         if (validationEnabled())
131             verifyModule(m_ftlState.module);
132     }
133
134 private:
135     
136     void createPhiVariables()
137     {
138         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
139             BasicBlock* block = m_graph.block(blockIndex);
140             if (!block)
141                 continue;
142             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
143                 Node* node = block->at(nodeIndex);
144                 if (node->op() != Phi)
145                     continue;
146                 LType type;
147                 switch (node->flags() & NodeResultMask) {
148                 case NodeResultNumber:
149                     type = m_out.doubleType;
150                     break;
151                 case NodeResultInt32:
152                     type = m_out.int32;
153                     break;
154                 case NodeResultBoolean:
155                     type = m_out.boolean;
156                     break;
157                 case NodeResultJS:
158                     type = m_out.int64;
159                     break;
160                 default:
161                     RELEASE_ASSERT_NOT_REACHED();
162                     break;
163                 }
164                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
165             }
166         }
167     }
168     
169     void compileBlock(BasicBlock* block)
170     {
171         if (!block)
172             return;
173         
174         if (verboseCompilationEnabled())
175             dataLog("Compiling block ", *block, "\n");
176         
177         m_highBlock = block;
178         
179         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
180         
181         m_nextHighBlock = 0;
182         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
183             m_nextHighBlock = m_graph.block(nextBlockIndex);
184             if (m_nextHighBlock)
185                 break;
186         }
187         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
188         
189         // All of this effort to find the next block gives us the ability to keep the
190         // generated IR in roughly program order. This ought not affect the performance
191         // of the generated code (since we expect LLVM to reorder things) but it will
192         // make IR dumps easier to read.
193         m_out.appendTo(lowBlock, m_nextLowBlock);
194         
195         if (Options::ftlCrashes())
196             m_out.crashNonTerminal();
197         
198         if (!m_highBlock->cfaHasVisited) {
199             m_out.crash();
200             return;
201         }
202         
203         initializeOSRExitStateForBlock();
204         
205         m_live = block->ssa->liveAtHead;
206         
207         m_state.reset();
208         m_state.beginBasicBlock(m_highBlock);
209         
210         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
211             if (!compileNode(m_nodeIndex))
212                 break;
213         }
214     }
215     
216     bool compileNode(unsigned nodeIndex)
217     {
218         if (!m_state.isValid()) {
219             m_out.unreachable();
220             return false;
221         }
222         
223         m_node = m_highBlock->at(nodeIndex);
224         m_codeOriginForExitProfile = m_node->codeOrigin;
225         m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
226         
227         if (verboseCompilationEnabled())
228             dataLog("Lowering ", m_node, "\n");
229         
230         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
231         
232         m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
233         
234         switch (m_node->op()) {
235         case Upsilon:
236             compileUpsilon();
237             break;
238         case Phi:
239             compilePhi();
240             break;
241         case JSConstant:
242             compileJSConstant();
243             break;
244         case WeakJSConstant:
245             compileWeakJSConstant();
246             break;
247         case GetArgument:
248             compileGetArgument();
249             break;
250         case ExtractOSREntryLocal:
251             compileExtractOSREntryLocal();
252             break;
253         case GetLocal:
254             compileGetLocal();
255             break;
256         case SetLocal:
257             compileSetLocal();
258             break;
259         case MovHint:
260             compileMovHint();
261             break;
262         case ZombieHint:
263             compileZombieHint();
264             break;
265         case MovHintAndCheck:
266             compileMovHintAndCheck();
267             break;
268         case Phantom:
269             compilePhantom();
270             break;
271         case Flush:
272         case PhantomLocal:
273         case SetArgument:
274         case LoopHint:
275             break;
276         case ArithAdd:
277         case ValueAdd:
278             compileAdd();
279             break;
280         case ArithSub:
281             compileArithSub();
282             break;
283         case ArithMul:
284             compileArithMul();
285             break;
286         case ArithDiv:
287             compileArithDiv();
288             break;
289         case ArithMod:
290             compileArithMod();
291             break;
292         case ArithMin:
293         case ArithMax:
294             compileArithMinOrMax();
295             break;
296         case ArithAbs:
297             compileArithAbs();
298             break;
299         case ArithNegate:
300             compileArithNegate();
301             break;
302         case BitAnd:
303             compileBitAnd();
304             break;
305         case BitOr:
306             compileBitOr();
307             break;
308         case BitXor:
309             compileBitXor();
310             break;
311         case BitRShift:
312             compileBitRShift();
313             break;
314         case BitLShift:
315             compileBitLShift();
316             break;
317         case BitURShift:
318             compileBitURShift();
319             break;
320         case UInt32ToNumber:
321             compileUInt32ToNumber();
322             break;
323         case Int32ToDouble:
324             compileInt32ToDouble();
325             break;
326         case CheckStructure:
327             compileCheckStructure();
328             break;
329         case StructureTransitionWatchpoint:
330             compileStructureTransitionWatchpoint();
331             break;
332         case ArrayifyToStructure:
333             compileArrayifyToStructure();
334             break;
335         case PutStructure:
336             compilePutStructure();
337             break;
338         case PhantomPutStructure:
339             compilePhantomPutStructure();
340             break;
341         case GetButterfly:
342             compileGetButterfly();
343             break;
344         case GetArrayLength:
345             compileGetArrayLength();
346             break;
347         case GetByVal:
348             compileGetByVal();
349             break;
350         case PutByVal:
351         case PutByValAlias:
352             compilePutByVal();
353             break;
354         case GetByOffset:
355             compileGetByOffset();
356             break;
357         case PutByOffset:
358             compilePutByOffset();
359             break;
360         case GetGlobalVar:
361             compileGetGlobalVar();
362             break;
363         case PutGlobalVar:
364             compilePutGlobalVar();
365             break;
366         case CompareEq:
367             compileCompareEq();
368             break;
369         case CompareEqConstant:
370             compileCompareEqConstant();
371             break;
372         case CompareStrictEq:
373             compileCompareStrictEq();
374             break;
375         case CompareStrictEqConstant:
376             compileCompareStrictEqConstant();
377             break;
378         case CompareLess:
379             compileCompareLess();
380             break;
381         case CompareLessEq:
382             compileCompareLessEq();
383             break;
384         case CompareGreater:
385             compileCompareGreater();
386             break;
387         case CompareGreaterEq:
388             compileCompareGreaterEq();
389             break;
390         case LogicalNot:
391             compileLogicalNot();
392             break;
393         case Jump:
394             compileJump();
395             break;
396         case Branch:
397             compileBranch();
398             break;
399         case Switch:
400             compileSwitch();
401             break;
402         case Return:
403             compileReturn();
404             break;
405         case ForceOSRExit:
406             compileForceOSRExit();
407             break;
408         default:
409             RELEASE_ASSERT_NOT_REACHED();
410             break;
411         }
412         
413         if (m_node->shouldGenerate())
414             DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
415         
416         if (m_node->adjustedRefCount())
417             m_live.add(m_node);
418         
419         if (shouldExecuteEffects)
420             m_interpreter.executeEffects(nodeIndex);
421         
422         return true;
423     }
424     
425     void compileUpsilon()
426     {
427         LValue destination = m_phis.get(m_node->phi());
428         
429         switch (m_node->child1().useKind()) {
430         case NumberUse:
431             m_out.set(lowDouble(m_node->child1()), destination);
432             break;
433         case Int32Use:
434             m_out.set(lowInt32(m_node->child1()), destination);
435             break;
436         case BooleanUse:
437             m_out.set(lowBoolean(m_node->child1()), destination);
438             break;
439         case CellUse:
440             m_out.set(lowCell(m_node->child1()), destination);
441             break;
442         case UntypedUse:
443             m_out.set(lowJSValue(m_node->child1()), destination);
444             break;
445         default:
446             RELEASE_ASSERT_NOT_REACHED();
447             break;
448         }
449     }
450     
451     void compilePhi()
452     {
453         LValue source = m_phis.get(m_node);
454         
455         switch (m_node->flags() & NodeResultMask) {
456         case NodeResultNumber:
457             setDouble(m_out.get(source));
458             break;
459         case NodeResultInt32:
460             setInt32(m_out.get(source));
461             break;
462         case NodeResultBoolean:
463             setBoolean(m_out.get(source));
464             break;
465         case NodeResultJS:
466             setJSValue(m_out.get(source));
467             break;
468         default:
469             RELEASE_ASSERT_NOT_REACHED();
470             break;
471         }
472     }
473     
474     void compileJSConstant()
475     {
476         JSValue value = m_graph.valueOfJSConstant(m_node);
477         if (value.isDouble())
478             setDouble(m_out.constDouble(value.asDouble()));
479         else
480             setJSValue(m_out.constInt64(JSValue::encode(value)));
481     }
482     
483     void compileWeakJSConstant()
484     {
485         setJSValue(weakPointer(m_node->weakConstant()));
486     }
487     
488     void compileGetArgument()
489     {
490         VariableAccessData* variable = m_node->variableAccessData();
491         int operand = variable->operand();
492
493         LValue jsValue = m_out.load64(addressFor(operand));
494
495         switch (useKindFor(variable->flushFormat())) {
496         case Int32Use:
497             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
498             setInt32(unboxInt32(jsValue));
499             break;
500         case CellUse:
501             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
502             setJSValue(jsValue);
503             break;
504         case BooleanUse:
505             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
506             setBoolean(unboxBoolean(jsValue));
507             break;
508         case UntypedUse:
509             setJSValue(jsValue);
510             break;
511         default:
512             RELEASE_ASSERT_NOT_REACHED();
513             break;
514         }
515     }
516     
517     void compileExtractOSREntryLocal()
518     {
519         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
520             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
521         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal())));
522     }
523     
524     void compileGetLocal()
525     {
526         // GetLocals arise only for captured variables.
527         
528         VariableAccessData* variable = m_node->variableAccessData();
529         AbstractValue& value = m_state.variables().operand(variable->local());
530         
531         RELEASE_ASSERT(variable->isCaptured());
532         
533         if (isInt32Speculation(value.m_type))
534             setInt32(m_out.load32(payloadFor(variable->local())));
535         else
536             setJSValue(m_out.load64(addressFor(variable->local())));
537     }
538     
539     void compileSetLocal()
540     {
541         observeMovHint(m_node);
542         
543         VariableAccessData* variable = m_node->variableAccessData();
544         SpeculatedType prediction = variable->argumentAwarePrediction();
545         
546         if (variable->shouldUnboxIfPossible()) {
547             if (variable->shouldUseDoubleFormat()) {
548                 LValue value = lowDouble(m_node->child1());
549                 m_out.storeDouble(value, addressFor(variable->local()));
550                 m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
551                 return;
552             }
553             
554             if (isInt32Speculation(prediction)) {
555                 LValue value = lowInt32(m_node->child1());
556                 m_out.store32(value, payloadFor(variable->local()));
557                 m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
558                 return;
559             }
560             if (isCellSpeculation(prediction)) {
561                 LValue value = lowCell(m_node->child1());
562                 m_out.store64(value, addressFor(variable->local()));
563                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
564                 return;
565             }
566             if (isBooleanSpeculation(prediction)) {
567                 speculateBoolean(m_node->child1());
568                 m_out.store64(
569                     lowJSValue(m_node->child1(), ManualOperandSpeculation),
570                     addressFor(variable->local()));
571                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
572                 return;
573             }
574         }
575         
576         LValue value = lowJSValue(m_node->child1());
577         m_out.store64(value, addressFor(variable->local()));
578         m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
579     }
580     
581     void compileMovHint()
582     {
583         observeMovHint(m_node);
584     }
585     
586     void compileZombieHint()
587     {
588         VariableAccessData* data = m_node->variableAccessData();
589         m_lastSetOperand = data->local();
590         m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
591     }
592     
593     void compileMovHintAndCheck()
594     {
595         observeMovHint(m_node);
596         speculate(m_node->child1());
597     }
598     
599     void compilePhantom()
600     {
601         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
602     }
603     
604     void compileAdd()
605     {
606         switch (m_node->binaryUseKind()) {
607         case Int32Use: {
608             LValue left = lowInt32(m_node->child1());
609             LValue right = lowInt32(m_node->child2());
610             
611             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
612                 setInt32(m_out.add(left, right));
613                 break;
614             }
615             
616             LValue result = m_out.addWithOverflow32(left, right);
617             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
618             setInt32(m_out.extractValue(result, 0));
619             break;
620         }
621             
622         case NumberUse: {
623             setDouble(
624                 m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
625             break;
626         }
627             
628         default:
629             RELEASE_ASSERT_NOT_REACHED();
630             break;
631         }
632     }
633     
634     void compileArithSub()
635     {
636         switch (m_node->binaryUseKind()) {
637         case Int32Use: {
638             LValue left = lowInt32(m_node->child1());
639             LValue right = lowInt32(m_node->child2());
640             
641             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
642                 setInt32(m_out.sub(left, right));
643                 break;
644             }
645             
646             LValue result = m_out.subWithOverflow32(left, right);
647             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
648             setInt32(m_out.extractValue(result, 0));
649             break;
650         }
651             
652         case NumberUse: {
653             setDouble(
654                 m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
655             break;
656         }
657             
658         default:
659             RELEASE_ASSERT_NOT_REACHED();
660             break;
661         }
662     }
663     
664     void compileArithMul()
665     {
666         switch (m_node->binaryUseKind()) {
667         case Int32Use: {
668             LValue left = lowInt32(m_node->child1());
669             LValue right = lowInt32(m_node->child2());
670             
671             LValue result;
672             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
673                 result = m_out.mul(left, right);
674             else {
675                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
676                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
677                 result = m_out.extractValue(overflowResult, 0);
678             }
679             
680             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
681                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
682                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
683                 
684                 m_out.branch(m_out.notZero32(result), continuation, slowCase);
685                 
686                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
687                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
688                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
689                 m_out.jump(continuation);
690                 m_out.appendTo(continuation, lastNext);
691             }
692             
693             setInt32(result);
694             break;
695         }
696             
697         case NumberUse: {
698             setDouble(
699                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
700             break;
701         }
702             
703         default:
704             RELEASE_ASSERT_NOT_REACHED();
705             break;
706         }
707     }
708     
709     void compileArithDiv()
710     {
711         switch (m_node->binaryUseKind()) {
712         case Int32Use: {
713             LValue numerator = lowInt32(m_node->child1());
714             LValue denominator = lowInt32(m_node->child2());
715             
716             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
717             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
718             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
719             
720             Vector<ValueFromBlock, 3> results;
721             
722             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
723             
724             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
725             
726             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
727             
728             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
729             
730             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
731                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
732                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
733                 m_out.jump(continuation);
734             } else {
735                 // This is the case where we convert the result to an int after we're done. So,
736                 // if the denominator is zero, then the result should be result should be zero.
737                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
738                 // check above) and the numerator is -2^31 then the result should be -2^31.
739                 
740                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
741                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
742                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
743                 
744                 m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
745                 
746                 m_out.appendTo(divByZero, notDivByZero);
747                 results.append(m_out.anchor(m_out.int32Zero));
748                 m_out.jump(done);
749                 
750                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
751                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
752                 
753                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
754                 results.append(m_out.anchor(neg2ToThe31));
755                 m_out.jump(done);
756             }
757             
758             m_out.appendTo(continuation, done);
759             
760             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
761                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
762                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
763                 
764                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
765                 
766                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
767                 
768                 speculate(
769                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
770                 
771                 m_out.jump(numeratorContinuation);
772                 
773                 m_out.appendTo(numeratorContinuation, innerLastNext);
774             }
775             
776             LValue divisionResult = m_out.div(numerator, denominator);
777             
778             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
779                 speculate(
780                     Overflow, noValue(), 0,
781                     m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
782             }
783             
784             results.append(m_out.anchor(divisionResult));
785             m_out.jump(done);
786             
787             m_out.appendTo(done, lastNext);
788             
789             setInt32(m_out.phi(m_out.int32, results));
790             break;
791         }
792             
793         case NumberUse: {
794             setDouble(
795                 m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
796             break;
797         }
798             
799         default:
800             RELEASE_ASSERT_NOT_REACHED();
801             break;
802         }
803     }
804     
805     void compileArithMod()
806     {
807         switch (m_node->binaryUseKind()) {
808         case Int32Use: {
809             LValue numerator = lowInt32(m_node->child1());
810             LValue denominator = lowInt32(m_node->child2());
811             
812             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
813             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
814             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
815             
816             Vector<ValueFromBlock, 3> results;
817             
818             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
819             
820             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
821             
822             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
823             
824             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
825             
826             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
827             // separate case for that. But it probably doesn't matter so much.
828             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
829                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
830                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
831                 m_out.jump(continuation);
832             } else {
833                 // This is the case where we convert the result to an int after we're done. So,
834                 // if the denominator is zero, then the result should be result should be zero.
835                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
836                 // check above) and the numerator is -2^31 then the result should be -2^31.
837                 
838                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
839                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
840                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
841                 
842                 m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
843                 
844                 m_out.appendTo(modByZero, notModByZero);
845                 results.append(m_out.anchor(m_out.int32Zero));
846                 m_out.jump(done);
847                 
848                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
849                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
850                 
851                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
852                 results.append(m_out.anchor(m_out.int32Zero));
853                 m_out.jump(done);
854             }
855             
856             m_out.appendTo(continuation, done);
857             
858             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
859                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod zero numerator"));
860                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
861                 
862                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
863                 
864                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
865                 
866                 speculate(
867                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
868                 
869                 m_out.jump(numeratorContinuation);
870                 
871                 m_out.appendTo(numeratorContinuation, innerLastNext);
872             }
873             
874             results.append(m_out.anchor(m_out.rem(numerator, denominator)));
875             m_out.jump(done);
876             
877             m_out.appendTo(done, lastNext);
878             
879             setInt32(m_out.phi(m_out.int32, results));
880             break;
881         }
882             
883         case NumberUse: {
884             setDouble(
885                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
886             break;
887         }
888             
889         default:
890             RELEASE_ASSERT_NOT_REACHED();
891             break;
892         }
893     }
894
895     void compileArithMinOrMax()
896     {
897         switch (m_node->binaryUseKind()) {
898         case Int32Use: {
899             LValue left = lowInt32(m_node->child1());
900             LValue right = lowInt32(m_node->child2());
901             
902             setInt32(
903                 m_out.select(
904                     m_node->op() == ArithMin
905                         ? m_out.lessThan(left, right)
906                         : m_out.lessThan(right, left),
907                     left, right));
908             break;
909         }
910             
911         case NumberUse: {
912             LValue left = lowDouble(m_node->child1());
913             LValue right = lowDouble(m_node->child2());
914             
915             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
916             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
917             
918             Vector<ValueFromBlock, 2> results;
919             
920             results.append(m_out.anchor(left));
921             m_out.branch(
922                 m_node->op() == ArithMin
923                     ? m_out.doubleLessThan(left, right)
924                     : m_out.doubleGreaterThan(left, right),
925                 continuation, notLessThan);
926             
927             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
928             results.append(m_out.anchor(m_out.select(
929                 m_node->op() == ArithMin
930                     ? m_out.doubleGreaterThanOrEqual(left, right)
931                     : m_out.doubleLessThanOrEqual(left, right),
932                 right, m_out.constDouble(0.0 / 0.0))));
933             m_out.jump(continuation);
934             
935             m_out.appendTo(continuation, lastNext);
936             setDouble(m_out.phi(m_out.doubleType, results));
937             break;
938         }
939             
940         default:
941             RELEASE_ASSERT_NOT_REACHED();
942             break;
943         }
944     }
945     
946     void compileArithAbs()
947     {
948         switch (m_node->child1().useKind()) {
949         case Int32Use: {
950             LValue value = lowInt32(m_node->child1());
951             
952             LValue mask = m_out.aShr(value, m_out.constInt32(31));
953             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
954             
955             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
956             
957             setInt32(result);
958             break;
959         }
960             
961         case NumberUse: {
962             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
963             break;
964         }
965             
966         default:
967             RELEASE_ASSERT_NOT_REACHED();
968             break;
969         }
970     }
971     
972     void compileArithNegate()
973     {
974         switch (m_node->child1().useKind()) {
975         case Int32Use: {
976             LValue value = lowInt32(m_node->child1());
977             
978             LValue result;
979             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
980                 result = m_out.neg(value);
981             else {
982                 // We don't have a negate-with-overflow intrinsic. Hopefully this
983                 // does the trick, though.
984                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
985                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
986                 result = m_out.extractValue(overflowResult, 0);
987             }
988             
989             setInt32(result);
990             break;
991         }
992             
993         case NumberUse: {
994             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
995             break;
996         }
997             
998         default:
999             RELEASE_ASSERT_NOT_REACHED();
1000             break;
1001         }
1002     }
1003     
1004     void compileBitAnd()
1005     {
1006         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1007     }
1008     
1009     void compileBitOr()
1010     {
1011         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1012     }
1013     
1014     void compileBitXor()
1015     {
1016         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1017     }
1018     
1019     void compileBitRShift()
1020     {
1021         setInt32(m_out.aShr(
1022             lowInt32(m_node->child1()),
1023             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1024     }
1025     
1026     void compileBitLShift()
1027     {
1028         setInt32(m_out.shl(
1029             lowInt32(m_node->child1()),
1030             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1031     }
1032     
1033     void compileBitURShift()
1034     {
1035         setInt32(m_out.lShr(
1036             lowInt32(m_node->child1()),
1037             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1038     }
1039     
1040     void compileUInt32ToNumber()
1041     {
1042         LValue value = lowInt32(m_node->child1());
1043
1044         if (!nodeCanSpeculateInteger(m_node->arithNodeFlags())) {
1045             setDouble(m_out.unsignedToDouble(value));
1046             return;
1047         }
1048         
1049         speculateForward(
1050             Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
1051             FormattedValue(ValueFormatUInt32, value));
1052         setInt32(value);
1053     }
1054     
1055     void compileInt32ToDouble()
1056     {
1057         // This node is tricky to compile in the DFG backend because it tries to
1058         // avoid converting child1 to a double in-place, as that would make subsequent
1059         // int uses of of child1 fail. But the FTL needs no such special magic, since
1060         // unlike the DFG backend, the FTL allows each node to have multiple
1061         // contemporaneous low-level representations. So, this gives child1 a double
1062         // representation and then forwards that representation to m_node.
1063         
1064         setDouble(lowDouble(m_node->child1()));
1065     }
1066     
1067     void compileCheckStructure()
1068     {
1069         LValue cell = lowCell(m_node->child1());
1070         
1071         ExitKind exitKind;
1072         if (m_node->child1()->op() == WeakJSConstant)
1073             exitKind = BadWeakConstantCache;
1074         else
1075             exitKind = BadCache;
1076         
1077         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1078         
1079         if (m_node->structureSet().size() == 1) {
1080             speculate(
1081                 exitKind, jsValueValue(cell), 0,
1082                 m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
1083             return;
1084         }
1085         
1086         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1087         
1088         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1089         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1090             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1091             m_out.branch(
1092                 m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
1093                 continuation, nextStructure);
1094             m_out.appendTo(nextStructure);
1095         }
1096         
1097         speculate(
1098             exitKind, jsValueValue(cell), 0,
1099             m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
1100         
1101         m_out.jump(continuation);
1102         m_out.appendTo(continuation, lastNext);
1103     }
1104     
1105     void compileStructureTransitionWatchpoint()
1106     {
1107         addWeakReference(m_node->structure());
1108         
1109         // FIXME: Implement structure transition watchpoints.
1110         // https://bugs.webkit.org/show_bug.cgi?id=113647
1111         
1112         speculateCell(m_node->child1());
1113     }
1114     
1115     void compileArrayifyToStructure()
1116     {
1117         LValue cell = lowCell(m_node->child1());
1118         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1119         
1120         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1121         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1122         
1123         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1124         
1125         m_out.branch(
1126             m_out.notEqual(structure, weakPointer(m_node->structure())),
1127             unexpectedStructure, continuation);
1128         
1129         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1130         
1131         if (property) {
1132             switch (m_node->arrayMode().type()) {
1133             case Array::Int32:
1134             case Array::Double:
1135             case Array::Contiguous:
1136                 speculate(
1137                     Uncountable, noValue(), 0,
1138                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1139                 break;
1140             default:
1141                 break;
1142             }
1143         }
1144         
1145         switch (m_node->arrayMode().type()) {
1146         case Array::Int32:
1147             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1148             break;
1149         case Array::Double:
1150             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1151             break;
1152         case Array::Contiguous:
1153             if (m_node->arrayMode().conversion() == Array::RageConvert)
1154                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1155             else
1156                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1157             break;
1158         case Array::ArrayStorage:
1159         case Array::SlowPutArrayStorage:
1160             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1161             break;
1162         default:
1163             RELEASE_ASSERT_NOT_REACHED();
1164             break;
1165         }
1166         
1167         structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1168         speculate(
1169             BadIndexingType, jsValueValue(cell), 0,
1170             m_out.notEqual(structure, weakPointer(m_node->structure())));
1171         m_out.jump(continuation);
1172         
1173         m_out.appendTo(continuation, lastNext);
1174     }
1175     
1176     void compilePutStructure()
1177     {
1178         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1179         
1180         m_out.store64(
1181             m_out.constIntPtr(m_node->structureTransitionData().newStructure),
1182             lowCell(m_node->child1()), m_heaps.JSCell_structure);
1183     }
1184     
1185     void compilePhantomPutStructure()
1186     {
1187         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1188     }
1189     
1190     void compileGetButterfly()
1191     {
1192         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1193     }
1194     
1195     void compileGetArrayLength()
1196     {
1197         switch (m_node->arrayMode().type()) {
1198         case Array::Int32:
1199         case Array::Double:
1200         case Array::Contiguous: {
1201             setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1202             break;
1203         }
1204             
1205         default:
1206             RELEASE_ASSERT_NOT_REACHED();
1207             break;
1208         }
1209     }
1210     
1211     void compileGetByVal()
1212     {
1213         LValue index = lowInt32(m_node->child2());
1214         LValue storage = lowStorage(m_node->child3());
1215         
1216         switch (m_node->arrayMode().type()) {
1217         case Array::Int32:
1218         case Array::Contiguous: {
1219             if (m_node->arrayMode().isInBounds()) {
1220                 speculate(
1221                     OutOfBounds, noValue(), 0,
1222                     m_out.aboveOrEqual(
1223                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1224                 
1225                 LValue result = m_out.load64(m_out.baseIndex(
1226                     m_node->arrayMode().type() == Array::Int32 ?
1227                         m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1228                     storage, m_out.zeroExt(index, m_out.intPtr),
1229                     m_state.forNode(m_node->child2()).m_value));
1230                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1231                 setJSValue(result);
1232                 return;
1233             }
1234             
1235             // FIXME: Implement hole/OOB loads in the FTL.
1236             // https://bugs.webkit.org/show_bug.cgi?id=118077
1237             RELEASE_ASSERT_NOT_REACHED();
1238             return;
1239         }
1240             
1241         case Array::Double: {
1242             if (m_node->arrayMode().isInBounds()) {
1243                 if (m_node->arrayMode().isSaneChain()) {
1244                     // FIXME: Implement structure transition watchpoints.
1245                     // https://bugs.webkit.org/show_bug.cgi?id=113647
1246                 }
1247             
1248                 speculate(
1249                     OutOfBounds, noValue(), 0,
1250                     m_out.aboveOrEqual(
1251                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1252                 
1253                 LValue result = m_out.loadDouble(m_out.baseIndex(
1254                     m_heaps.indexedDoubleProperties,
1255                     storage, m_out.zeroExt(index, m_out.intPtr),
1256                     m_state.forNode(m_node->child2()).m_value));
1257                 
1258                 if (!m_node->arrayMode().isSaneChain()) {
1259                     speculate(
1260                         LoadFromHole, noValue(), 0,
1261                         m_out.doubleNotEqualOrUnordered(result, result));
1262                 }
1263                 setDouble(result);
1264                 break;
1265             }
1266             
1267             // FIXME: Implement hole/OOB loads in the FTL.
1268             // https://bugs.webkit.org/show_bug.cgi?id=118077
1269             RELEASE_ASSERT_NOT_REACHED();
1270             return;
1271         }
1272             
1273         default:
1274             RELEASE_ASSERT_NOT_REACHED();
1275             return;
1276         }
1277     }
1278     
1279     void compilePutByVal()
1280     {
1281         Edge child1 = m_graph.varArgChild(m_node, 0);
1282         Edge child2 = m_graph.varArgChild(m_node, 1);
1283         Edge child3 = m_graph.varArgChild(m_node, 2);
1284         Edge child4 = m_graph.varArgChild(m_node, 3);
1285
1286         LValue base = lowCell(child1);
1287         LValue index = lowInt32(child2);
1288         LValue storage = lowStorage(child4);
1289         
1290         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
1291         LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
1292             
1293         switch (m_node->arrayMode().type()) {
1294         case Array::Int32:
1295         case Array::Contiguous: {
1296             LValue value = lowJSValue(child3, ManualOperandSpeculation);
1297             
1298             if (m_node->arrayMode().type() == Array::Int32)
1299                 FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
1300             
1301             TypedPointer elementPointer = m_out.baseIndex(
1302                 m_node->arrayMode().type() == Array::Int32 ?
1303                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1304                 storage, m_out.zeroExt(index, m_out.intPtr),
1305                 m_state.forNode(child2).m_value);
1306             
1307             if (m_node->op() == PutByValAlias) {
1308                 m_out.store64(value, elementPointer);
1309                 break;
1310             }
1311             
1312             contiguousPutByValOutOfBounds(
1313                 codeBlock()->isStrictMode()
1314                     ? operationPutByValBeyondArrayBoundsStrict
1315                     : operationPutByValBeyondArrayBoundsNonStrict,
1316                 base, storage, index, value, continuation);
1317             
1318             m_out.store64(value, elementPointer);
1319             break;
1320         }
1321             
1322         case Array::Double: {
1323             LValue value = lowDouble(child3);
1324             
1325             FTL_TYPE_CHECK(
1326                 doubleValue(value), child3, SpecRealNumber,
1327                 m_out.doubleNotEqualOrUnordered(value, value));
1328             
1329             TypedPointer elementPointer = m_out.baseIndex(
1330                 m_heaps.indexedDoubleProperties,
1331                 storage, m_out.zeroExt(index, m_out.intPtr),
1332                 m_state.forNode(child2).m_value);
1333
1334             if (m_node->op() == PutByValAlias) {
1335                 m_out.storeDouble(value, elementPointer);
1336                 break;
1337             }
1338             
1339             contiguousPutByValOutOfBounds(
1340                 codeBlock()->isStrictMode()
1341                     ? operationPutDoubleByValBeyondArrayBoundsStrict
1342                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1343                 base, storage, index, value, continuation);
1344             
1345             m_out.storeDouble(value, elementPointer);
1346             break;
1347         }
1348             
1349         default:
1350             RELEASE_ASSERT_NOT_REACHED();
1351             break;
1352         }
1353
1354         m_out.jump(continuation);
1355         m_out.appendTo(continuation, outerLastNext);
1356     }
1357     
1358     void compileGetByOffset()
1359     {
1360         StorageAccessData& data =
1361             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1362         
1363         setJSValue(
1364             m_out.load64(
1365                 m_out.address(
1366                     m_heaps.properties[data.identifierNumber],
1367                     lowStorage(m_node->child1()),
1368                     offsetRelativeToBase(data.offset))));
1369     }
1370     
1371     void compilePutByOffset()
1372     {
1373         StorageAccessData& data =
1374             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1375         
1376         m_out.store64(
1377             lowJSValue(m_node->child3()),
1378             m_out.address(
1379                 m_heaps.properties[data.identifierNumber],
1380                 lowStorage(m_node->child1()),
1381                 offsetRelativeToBase(data.offset)));
1382     }
1383     
1384     void compileGetGlobalVar()
1385     {
1386         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
1387     }
1388     
1389     void compilePutGlobalVar()
1390     {
1391         m_out.store64(
1392             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
1393     }
1394     
1395     void compileCompareEq()
1396     {
1397         if (m_node->isBinaryUseKind(Int32Use)
1398             || m_node->isBinaryUseKind(NumberUse)
1399             || m_node->isBinaryUseKind(ObjectUse)) {
1400             compileCompareStrictEq();
1401             return;
1402         }
1403         
1404         RELEASE_ASSERT_NOT_REACHED();
1405     }
1406     
1407     void compileCompareEqConstant()
1408     {
1409         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
1410         masqueradesAsUndefinedWatchpointIfIsStillValid();
1411         setBoolean(
1412             equalNullOrUndefined(
1413                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
1414     }
1415     
1416     void compileCompareStrictEq()
1417     {
1418         if (m_node->isBinaryUseKind(Int32Use)) {
1419             setBoolean(
1420                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1421             return;
1422         }
1423         
1424         if (m_node->isBinaryUseKind(NumberUse)) {
1425             setBoolean(
1426                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1427         }
1428         
1429         if (m_node->isBinaryUseKind(ObjectUse)) {
1430             masqueradesAsUndefinedWatchpointIfIsStillValid();
1431             setBoolean(
1432                 m_out.equal(
1433                     lowNonNullObject(m_node->child1()),
1434                     lowNonNullObject(m_node->child2())));
1435             return;
1436         }
1437         
1438         RELEASE_ASSERT_NOT_REACHED();
1439     }
1440     
1441     void compileCompareStrictEqConstant()
1442     {
1443         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
1444
1445         if (constant.isUndefinedOrNull()
1446             && !masqueradesAsUndefinedWatchpointIfIsStillValid()) {
1447             if (constant.isNull()) {
1448                 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull));
1449                 return;
1450             }
1451         
1452             ASSERT(constant.isUndefined());
1453             setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
1454             return;
1455         }
1456         
1457         setBoolean(
1458             m_out.equal(
1459                 lowJSValue(m_node->child1()),
1460                 m_out.constInt64(JSValue::encode(constant))));
1461     }
1462     
1463     void compileCompareLess()
1464     {
1465         if (m_node->isBinaryUseKind(Int32Use)) {
1466             setBoolean(
1467                 m_out.lessThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1468             return;
1469         }
1470         
1471         if (m_node->isBinaryUseKind(NumberUse)) {
1472             setBoolean(
1473                 m_out.doubleLessThan(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1474             return;
1475         }
1476         
1477         RELEASE_ASSERT_NOT_REACHED();
1478     }
1479     
1480     void compileCompareLessEq()
1481     {
1482         if (m_node->isBinaryUseKind(Int32Use)) {
1483             setBoolean(
1484                 m_out.lessThanOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1485             return;
1486         }
1487         
1488         if (m_node->isBinaryUseKind(NumberUse)) {
1489             setBoolean(
1490                 m_out.doubleLessThanOrEqual(
1491                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1492             return;
1493         }
1494         
1495         RELEASE_ASSERT_NOT_REACHED();
1496     }
1497     
1498     void compileCompareGreater()
1499     {
1500         if (m_node->isBinaryUseKind(Int32Use)) {
1501             setBoolean(
1502                 m_out.greaterThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1503             return;
1504         }
1505         
1506         if (m_node->isBinaryUseKind(NumberUse)) {
1507             setBoolean(
1508                 m_out.doubleGreaterThan(
1509                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1510             return;
1511         }
1512         
1513         RELEASE_ASSERT_NOT_REACHED();
1514     }
1515     
1516     void compileCompareGreaterEq()
1517     {
1518         if (m_node->isBinaryUseKind(Int32Use)) {
1519             setBoolean(
1520                 m_out.greaterThanOrEqual(
1521                     lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1522             return;
1523         }
1524         
1525         if (m_node->isBinaryUseKind(NumberUse)) {
1526             setBoolean(
1527                 m_out.doubleGreaterThanOrEqual(
1528                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1529             return;
1530         }
1531         
1532         RELEASE_ASSERT_NOT_REACHED();
1533     }
1534     
1535     void compileLogicalNot()
1536     {
1537         setBoolean(m_out.bitNot(boolify(m_node->child1())));
1538     }
1539     
1540     void compileJump()
1541     {
1542         m_out.jump(lowBlock(m_node->takenBlock()));
1543     }
1544     
1545     void compileBranch()
1546     {
1547         m_out.branch(
1548             boolify(m_node->child1()),
1549             lowBlock(m_node->takenBlock()),
1550             lowBlock(m_node->notTakenBlock()));
1551     }
1552     
1553     void compileSwitch()
1554     {
1555         SwitchData* data = m_node->switchData();
1556         switch (data->kind) {
1557         case SwitchImm: {
1558             Vector<ValueFromBlock, 2> intValues;
1559             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
1560             
1561             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
1562             
1563             switch (m_node->child1().useKind()) {
1564             case Int32Use: {
1565                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
1566                 m_out.jump(switchOnInts);
1567                 break;
1568             }
1569                 
1570             case UntypedUse: {
1571                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
1572                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
1573                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
1574                 
1575                 LValue boxedValue = lowJSValue(m_node->child1());
1576                 m_out.branch(isNotInt32(boxedValue), isNotInt, isInt);
1577                 
1578                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
1579                 
1580                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
1581                 m_out.jump(switchOnInts);
1582                 
1583                 m_out.appendTo(isNotInt, isDouble);
1584                 m_out.branch(
1585                     isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble);
1586                 
1587                 m_out.appendTo(isDouble, innerLastNext);
1588                 LValue doubleValue = unboxDouble(boxedValue);
1589                 LValue intInDouble = m_out.fpToInt32(doubleValue);
1590                 intValues.append(m_out.anchor(intInDouble));
1591                 m_out.branch(
1592                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
1593                     switchOnInts, lowBlock(data->fallThrough));
1594                 break;
1595             }
1596                 
1597             default:
1598                 RELEASE_ASSERT_NOT_REACHED();
1599                 break;
1600             }
1601             
1602             m_out.appendTo(switchOnInts, lastNext);
1603             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
1604             return;
1605         }
1606         
1607         case SwitchChar: {
1608             LValue stringValue;
1609             
1610             switch (m_node->child1().useKind()) {
1611             case StringUse: {
1612                 stringValue = lowString(m_node->child1());
1613                 break;
1614             }
1615                 
1616             case UntypedUse: {
1617                 LValue unboxedValue = lowJSValue(m_node->child1());
1618                 
1619                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
1620                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
1621                 
1622                 m_out.branch(
1623                     isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase);
1624                 
1625                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
1626                 LValue cellValue = unboxedValue;
1627                 m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase);
1628                 
1629                 m_out.appendTo(isStringCase, lastNext);
1630                 stringValue = cellValue;
1631                 break;
1632             }
1633                 
1634             default:
1635                 RELEASE_ASSERT_NOT_REACHED();
1636                 break;
1637             }
1638             
1639             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
1640             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
1641             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
1642             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
1643             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
1644             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
1645             
1646             m_out.branch(
1647                 m_out.notEqual(
1648                     m_out.load32(stringValue, m_heaps.JSString_length),
1649                     m_out.int32One),
1650                 lowBlock(data->fallThrough), lengthIs1);
1651             
1652             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
1653             Vector<ValueFromBlock, 2> values;
1654             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
1655             values.append(m_out.anchor(fastValue));
1656             m_out.branch(m_out.isNull(fastValue), needResolution, resolved);
1657             
1658             m_out.appendTo(needResolution, resolved);
1659             values.append(m_out.anchor(
1660                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
1661             m_out.jump(resolved);
1662             
1663             m_out.appendTo(resolved, is8Bit);
1664             LValue value = m_out.phi(m_out.intPtr, values);
1665             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
1666             m_out.branch(
1667                 m_out.testNonZero32(
1668                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
1669                     m_out.constInt32(StringImpl::flagIs8Bit())),
1670                 is8Bit, is16Bit);
1671             
1672             Vector<ValueFromBlock, 2> characters;
1673             m_out.appendTo(is8Bit, is16Bit);
1674             characters.append(m_out.anchor(
1675                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
1676             m_out.jump(continuation);
1677             
1678             m_out.appendTo(is16Bit, continuation);
1679             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
1680             m_out.jump(continuation);
1681             
1682             m_out.appendTo(continuation, lastNext);
1683             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
1684             return;
1685         }
1686         
1687         case SwitchString:
1688             RELEASE_ASSERT_NOT_REACHED();
1689             break;
1690         }
1691         
1692         RELEASE_ASSERT_NOT_REACHED();
1693     }
1694     
1695     void compileReturn()
1696     {
1697         // FIXME: have a real epilogue when we switch to using our calling convention.
1698         // https://bugs.webkit.org/show_bug.cgi?id=113621
1699         m_out.ret(lowJSValue(m_node->child1()));
1700     }
1701     
1702     void compileForceOSRExit()
1703     {
1704         terminate(InadequateCoverage);
1705     }
1706     
1707     LValue boolify(Edge edge)
1708     {
1709         switch (edge.useKind()) {
1710         case BooleanUse:
1711             return lowBoolean(m_node->child1());
1712         case Int32Use:
1713             return m_out.notZero32(lowInt32(m_node->child1()));
1714         case NumberUse:
1715             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
1716         case ObjectOrOtherUse:
1717             return m_out.bitNot(
1718                 equalNullOrUndefined(
1719                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
1720                     ManualOperandSpeculation));
1721         default:
1722             RELEASE_ASSERT_NOT_REACHED();
1723             return 0;
1724         }
1725     }
1726     
1727     enum StringOrObjectMode {
1728         AllCellsAreFalse,
1729         CellCaseSpeculatesObject
1730     };
1731     enum EqualNullOrUndefinedMode {
1732         EqualNull,
1733         EqualUndefined,
1734         EqualNullOrUndefined,
1735         SpeculateNullOrUndefined
1736     };
1737     LValue equalNullOrUndefined(
1738         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
1739         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
1740     {
1741         bool validWatchpoint = masqueradesAsUndefinedWatchpointIfIsStillValid();
1742         
1743         LValue value = lowJSValue(edge, operandMode);
1744         
1745         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
1746         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
1747         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
1748         
1749         m_out.branch(isNotCell(value), primitiveCase, cellCase);
1750         
1751         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
1752         
1753         Vector<ValueFromBlock, 3> results;
1754         
1755         switch (cellMode) {
1756         case AllCellsAreFalse:
1757             break;
1758         case CellCaseSpeculatesObject:
1759             FTL_TYPE_CHECK(
1760                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
1761                 m_out.equal(
1762                     m_out.loadPtr(value, m_heaps.JSCell_structure),
1763                     m_out.constIntPtr(vm().stringStructure.get())));
1764             break;
1765         }
1766         
1767         if (validWatchpoint) {
1768             results.append(m_out.anchor(m_out.booleanFalse));
1769             m_out.jump(continuation);
1770         } else {
1771             LBasicBlock masqueradesCase =
1772                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
1773                 
1774             LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure);
1775             
1776             results.append(m_out.anchor(m_out.booleanFalse));
1777             
1778             m_out.branch(
1779                 m_out.testNonZero8(
1780                     m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
1781                     m_out.constInt8(MasqueradesAsUndefined)),
1782                 masqueradesCase, continuation);
1783             
1784             m_out.appendTo(masqueradesCase, primitiveCase);
1785             
1786             results.append(m_out.anchor(
1787                 m_out.equal(
1788                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)),
1789                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
1790             m_out.jump(continuation);
1791         }
1792         
1793         m_out.appendTo(primitiveCase, continuation);
1794         
1795         LValue primitiveResult;
1796         switch (primitiveMode) {
1797         case EqualNull:
1798             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
1799             break;
1800         case EqualUndefined:
1801             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
1802             break;
1803         case EqualNullOrUndefined:
1804             primitiveResult = m_out.equal(
1805                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1806                 m_out.constInt64(ValueNull));
1807             break;
1808         case SpeculateNullOrUndefined:
1809             FTL_TYPE_CHECK(
1810                 jsValueValue(value), edge, SpecCell | SpecOther,
1811                 m_out.notEqual(
1812                     m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1813                     m_out.constInt64(ValueNull)));
1814             primitiveResult = m_out.booleanTrue;
1815             break;
1816         }
1817         results.append(m_out.anchor(primitiveResult));
1818         m_out.jump(continuation);
1819         
1820         m_out.appendTo(continuation, lastNext);
1821         
1822         return m_out.phi(m_out.boolean, results);
1823     }
1824     
1825     template<typename FunctionType>
1826     void contiguousPutByValOutOfBounds(
1827         FunctionType slowPathFunction,
1828         LValue base, LValue storage, LValue index, LValue value,
1829         LBasicBlock continuation)
1830     {
1831         LValue isNotInBounds = m_out.aboveOrEqual(
1832             index, m_out.load32(storage, m_heaps.Butterfly_publicLength));
1833         if (m_node->arrayMode().isInBounds())
1834             speculate(StoreToHoleOrOutOfBounds, noValue(), 0, isNotInBounds);
1835         else {
1836             LBasicBlock notInBoundsCase =
1837                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
1838             LBasicBlock performStore =
1839                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
1840                 
1841             m_out.branch(isNotInBounds, notInBoundsCase, performStore);
1842                 
1843             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
1844                 
1845             LValue isOutOfBounds = m_out.aboveOrEqual(
1846                 index, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
1847                 
1848             if (!m_node->arrayMode().isOutOfBounds())
1849                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
1850             else {
1851                 LBasicBlock outOfBoundsCase =
1852                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
1853                 LBasicBlock holeCase =
1854                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
1855                     
1856                 m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase);
1857                     
1858                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
1859                     
1860                 vmCall(
1861                     m_out.operation(slowPathFunction),
1862                     m_callFrame, base, index, value);
1863                     
1864                 m_out.jump(continuation);
1865                     
1866                 m_out.appendTo(holeCase, innerLastNext);
1867             }
1868             
1869             m_out.store32(
1870                 m_out.add(index, m_out.int32One),
1871                 storage, m_heaps.Butterfly_publicLength);
1872                 
1873             m_out.jump(performStore);
1874             m_out.appendTo(performStore, lastNext);
1875         }
1876     }
1877     
1878     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
1879     {
1880         Vector<SwitchCase> cases;
1881         for (unsigned i = 0; i < data->cases.size(); ++i) {
1882             cases.append(SwitchCase(
1883                 constInt(type, data->cases[i].value.switchLookupValue()),
1884                 lowBlock(data->cases[i].target)));
1885         }
1886         
1887         m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough));
1888     }
1889     
1890     void speculateBackward(
1891         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1892     {
1893         appendOSRExit(
1894             kind, lowValue, highValue, failCondition, BackwardSpeculation, FormattedValue());
1895     }
1896     
1897     void speculateForward(
1898         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
1899         const FormattedValue& recovery)
1900     {
1901         appendOSRExit(
1902             kind, lowValue, highValue, failCondition, ForwardSpeculation, recovery);
1903     }
1904     
1905     void speculate(
1906         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1907     {
1908         appendOSRExit(
1909             kind, lowValue, highValue, failCondition, m_direction, FormattedValue());
1910     }
1911     
1912     void terminate(ExitKind kind)
1913     {
1914         speculate(kind, noValue(), 0, m_out.booleanTrue);
1915     }
1916     
1917     void backwardTypeCheck(
1918         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1919         LValue failCondition)
1920     {
1921         appendTypeCheck(
1922             lowValue, highValue, typesPassedThrough, failCondition, BackwardSpeculation,
1923             FormattedValue());
1924     }
1925     
1926     void forwardTypeCheck(
1927         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1928         LValue failCondition)
1929     {
1930         appendTypeCheck(
1931             lowValue, highValue, typesPassedThrough, failCondition, ForwardSpeculation,
1932             FormattedValue());
1933     }
1934     
1935     void typeCheck(
1936         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1937         LValue failCondition)
1938     {
1939         appendTypeCheck(
1940             lowValue, highValue, typesPassedThrough, failCondition, m_direction,
1941             FormattedValue());
1942     }
1943     
1944     void appendTypeCheck(
1945         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1946         LValue failCondition, SpeculationDirection direction, FormattedValue recovery)
1947     {
1948         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
1949             return;
1950         ASSERT(mayHaveTypeCheck(highValue.useKind()));
1951         appendOSRExit(BadType, lowValue, highValue.node(), failCondition, direction, recovery);
1952         m_interpreter.filter(highValue, typesPassedThrough);
1953     }
1954     
1955     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1956     {
1957         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
1958         
1959         LoweredNodeValue value = m_int32Values.get(edge.node());
1960         if (isValid(value))
1961             return value.value();
1962         
1963         value = m_jsValueValues.get(edge.node());
1964         if (isValid(value)) {
1965             LValue boxedResult = value.value();
1966             FTL_TYPE_CHECK(
1967                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
1968             LValue result = unboxInt32(boxedResult);
1969             setInt32(edge.node(), result);
1970             return result;
1971         }
1972
1973         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
1974         terminate(Uncountable);
1975         return m_out.int32Zero;
1976     }
1977     
1978     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1979     {
1980         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
1981         
1982         LoweredNodeValue value = m_jsValueValues.get(edge.node());
1983         if (isValid(value)) {
1984             LValue uncheckedValue = value.value();
1985             FTL_TYPE_CHECK(
1986                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
1987             return uncheckedValue;
1988         }
1989         
1990         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
1991         terminate(Uncountable);
1992         return m_out.intPtrZero;
1993     }
1994     
1995     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1996     {
1997         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
1998         
1999         LValue result = lowCell(edge, mode);
2000         speculateObject(edge, result);
2001         return result;
2002     }
2003     
2004     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2005     {
2006         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse);
2007         
2008         LValue result = lowCell(edge, mode);
2009         speculateString(edge, result);
2010         return result;
2011     }
2012     
2013     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2014     {
2015         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2016         
2017         LValue result = lowCell(edge, mode);
2018         speculateNonNullObject(edge, result);
2019         return result;
2020     }
2021     
2022     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2023     {
2024         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
2025         
2026         LoweredNodeValue value = m_booleanValues.get(edge.node());
2027         if (isValid(value))
2028             return value.value();
2029         
2030         value = m_jsValueValues.get(edge.node());
2031         if (isValid(value)) {
2032             LValue unboxedResult = value.value();
2033             FTL_TYPE_CHECK(
2034                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
2035             LValue result = unboxBoolean(unboxedResult);
2036             setBoolean(edge.node(), result);
2037             return result;
2038         }
2039         
2040         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
2041         terminate(Uncountable);
2042         return m_out.booleanFalse;
2043     }
2044     
2045     LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2046     {
2047         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind()));
2048         
2049         LoweredNodeValue value = m_doubleValues.get(edge.node());
2050         if (isValid(value))
2051             return value.value();
2052         
2053         value = m_int32Values.get(edge.node());
2054         if (isValid(value)) {
2055             LValue result = m_out.intToDouble(value.value());
2056             setDouble(edge.node(), result);
2057             return result;
2058         }
2059         
2060         value = m_jsValueValues.get(edge.node());
2061         if (isValid(value)) {
2062             LValue boxedResult = value.value();
2063             
2064             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case"));
2065             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case"));
2066             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation"));
2067             
2068             m_out.branch(isNotInt32(boxedResult), doubleCase, intCase);
2069             
2070             LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
2071             
2072             ValueFromBlock intToDouble = m_out.anchor(
2073                 m_out.intToDouble(unboxInt32(boxedResult)));
2074             m_out.jump(continuation);
2075             
2076             m_out.appendTo(doubleCase, continuation);
2077             
2078             FTL_TYPE_CHECK(
2079                 jsValueValue(boxedResult), edge, SpecNumber, isCellOrMisc(boxedResult));
2080             
2081             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult));
2082             m_out.jump(continuation);
2083             
2084             m_out.appendTo(continuation, lastNext);
2085             
2086             LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
2087             
2088             setDouble(edge.node(), result);
2089             return result;
2090         }
2091         
2092         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecNumber));
2093         terminate(Uncountable);
2094         return m_out.doubleZero;
2095     }
2096     
2097     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2098     {
2099         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2100         
2101         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2102         if (isValid(value))
2103             return value.value();
2104         
2105         value = m_int32Values.get(edge.node());
2106         if (isValid(value)) {
2107             LValue result = boxInt32(value.value());
2108             setJSValue(edge.node(), result);
2109             return result;
2110         }
2111         
2112         value = m_booleanValues.get(edge.node());
2113         if (isValid(value)) {
2114             LValue result = boxBoolean(value.value());
2115             setJSValue(edge.node(), result);
2116             return result;
2117         }
2118         
2119         value = m_doubleValues.get(edge.node());
2120         if (isValid(value)) {
2121             LValue result = boxDouble(value.value());
2122             setJSValue(edge.node(), result);
2123             return result;
2124         }
2125         
2126         RELEASE_ASSERT_NOT_REACHED();
2127         return 0;
2128     }
2129     
2130     LValue lowStorage(Edge edge)
2131     {
2132         LoweredNodeValue value = m_storageValues.get(edge.node());
2133         if (isValid(value))
2134             return value.value();
2135         
2136         LValue result = lowCell(edge);
2137         setStorage(edge.node(), result);
2138         return result;
2139     }
2140     
2141     LValue isNotInt32(LValue jsValue)
2142     {
2143         return m_out.below(jsValue, m_tagTypeNumber);
2144     }
2145     LValue unboxInt32(LValue jsValue)
2146     {
2147         return m_out.castToInt32(jsValue);
2148     }
2149     LValue boxInt32(LValue value)
2150     {
2151         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
2152     }
2153     
2154     LValue isCellOrMisc(LValue jsValue)
2155     {
2156         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
2157     }
2158     LValue unboxDouble(LValue jsValue)
2159     {
2160         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
2161     }
2162     LValue boxDouble(LValue doubleValue)
2163     {
2164         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
2165     }
2166     
2167     LValue isNotCell(LValue jsValue)
2168     {
2169         return m_out.testNonZero64(jsValue, m_tagMask);
2170     }
2171     
2172     LValue isNotBoolean(LValue jsValue)
2173     {
2174         return m_out.testNonZero64(
2175             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
2176             m_out.constInt64(~1));
2177     }
2178     LValue unboxBoolean(LValue jsValue)
2179     {
2180         // We want to use a cast that guarantees that LLVM knows that even the integer
2181         // value is just 0 or 1. But for now we do it the dumb way.
2182         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
2183     }
2184     LValue boxBoolean(LValue value)
2185     {
2186         return m_out.select(
2187             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
2188     }
2189
2190     void speculate(Edge edge)
2191     {
2192         switch (edge.useKind()) {
2193         case UntypedUse:
2194             break;
2195         case KnownInt32Use:
2196         case KnownNumberUse:
2197             ASSERT(!m_interpreter.needsTypeCheck(edge));
2198             break;
2199         case Int32Use:
2200             speculateInt32(edge);
2201             break;
2202         case CellUse:
2203             speculateCell(edge);
2204             break;
2205         case KnownCellUse:
2206             ASSERT(!m_interpreter.needsTypeCheck(edge));
2207             break;
2208         case ObjectUse:
2209             speculateObject(edge);
2210             break;
2211         case StringUse:
2212             speculateString(edge);
2213             break;
2214         case RealNumberUse:
2215             speculateRealNumber(edge);
2216             break;
2217         case NumberUse:
2218             speculateNumber(edge);
2219             break;
2220         case BooleanUse:
2221             speculateBoolean(edge);
2222             break;
2223         default:
2224             RELEASE_ASSERT_NOT_REACHED();
2225         }
2226     }
2227     
2228     void speculate(Node*, Edge edge)
2229     {
2230         speculate(edge);
2231     }
2232     
2233     void speculateInt32(Edge edge)
2234     {
2235         lowInt32(edge);
2236     }
2237     
2238     void speculateCell(Edge edge)
2239     {
2240         lowCell(edge);
2241     }
2242     
2243     LValue isObject(LValue cell)
2244     {
2245         return m_out.notEqual(
2246             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2247             m_out.constIntPtr(vm().stringStructure.get()));
2248     }
2249     
2250     LValue isNotString(LValue cell)
2251     {
2252         return isObject(cell);
2253     }
2254     
2255     LValue isString(LValue cell)
2256     {
2257         return m_out.equal(
2258             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2259             m_out.constIntPtr(vm().stringStructure.get()));
2260     }
2261     
2262     LValue isNotObject(LValue cell)
2263     {
2264         return isString(cell);
2265     }
2266     
2267     void speculateObject(Edge edge, LValue cell)
2268     {
2269         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
2270     }
2271     
2272     void speculateObject(Edge edge)
2273     {
2274         speculateObject(edge, lowCell(edge));
2275     }
2276     
2277     void speculateString(Edge edge, LValue cell)
2278     {
2279         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
2280     }
2281     
2282     void speculateString(Edge edge)
2283     {
2284         speculateString(edge, lowCell(edge));
2285     }
2286     
2287     void speculateNonNullObject(Edge edge, LValue cell)
2288     {
2289         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
2290         FTL_TYPE_CHECK(
2291             jsValueValue(cell), edge, SpecObject, 
2292             m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get())));
2293         if (masqueradesAsUndefinedWatchpointIfIsStillValid())
2294             return;
2295         
2296         speculate(
2297             BadType, jsValueValue(cell), edge.node(),
2298             m_out.testNonZero8(
2299                 m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2300                 m_out.constInt8(MasqueradesAsUndefined)));
2301     }
2302     
2303     void speculateNumber(Edge edge)
2304     {
2305         // Do an early return here because lowDouble() can create a lot of control flow.
2306         if (!m_interpreter.needsTypeCheck(edge))
2307             return;
2308         
2309         lowDouble(edge);
2310     }
2311     
2312     void speculateRealNumber(Edge edge)
2313     {
2314         // Do an early return here because lowDouble() can create a lot of control flow.
2315         if (!m_interpreter.needsTypeCheck(edge))
2316             return;
2317         
2318         LValue value = lowDouble(edge);
2319         FTL_TYPE_CHECK(
2320             doubleValue(value), edge, SpecRealNumber,
2321             m_out.doubleNotEqualOrUnordered(value, value));
2322     }
2323     
2324     void speculateBoolean(Edge edge)
2325     {
2326         lowBoolean(edge);
2327     }
2328
2329     bool masqueradesAsUndefinedWatchpointIsStillValid()
2330     {
2331         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin);
2332     }
2333     
2334     bool masqueradesAsUndefinedWatchpointIfIsStillValid()
2335     {
2336         if (!masqueradesAsUndefinedWatchpointIsStillValid())
2337             return false;
2338         
2339         // FIXME: Implement masquerades-as-undefined watchpoints.
2340         // https://bugs.webkit.org/show_bug.cgi?id=113647
2341         return true;
2342     }
2343     
2344     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
2345     
2346     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
2347     {
2348         callPreflight();
2349         LValue result = m_out.call(function);
2350         callCheck(mode);
2351         return result;
2352     }
2353     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
2354     {
2355         callPreflight();
2356         LValue result = m_out.call(function, arg1);
2357         callCheck(mode);
2358         return result;
2359     }
2360     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
2361     {
2362         callPreflight();
2363         LValue result = m_out.call(function, arg1, arg2);
2364         callCheck(mode);
2365         return result;
2366     }
2367     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
2368     {
2369         callPreflight();
2370         LValue result = m_out.call(function, arg1, arg2, arg3);
2371         callCheck(mode);
2372         return result;
2373     }
2374     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
2375     {
2376         callPreflight();
2377         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
2378         callCheck(mode);
2379         return result;
2380     }
2381     
2382     void callPreflight(CodeOrigin codeOrigin)
2383     {
2384         m_out.store32(
2385             m_out.constInt32(
2386                 CallFrame::Location::encodeAsCodeOriginIndex(
2387                     codeBlock()->addCodeOrigin(codeOrigin))), 
2388             tagFor(JSStack::ArgumentCount));
2389     }
2390     void callPreflight()
2391     {
2392         callPreflight(m_node->codeOrigin);
2393     }
2394     
2395     void callCheck(ExceptionCheckMode mode = CheckExceptions)
2396     {
2397         if (mode == NoExceptions)
2398             return;
2399         
2400         LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception"));
2401         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
2402         
2403         m_out.branch(
2404             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
2405             didHaveException, continuation);
2406         
2407         LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation);
2408         // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622
2409         m_out.crash();
2410         
2411         m_out.appendTo(continuation, lastNext);
2412     }
2413     
2414     bool isLive(Node* node)
2415     {
2416         return m_live.contains(node);
2417     }
2418     
2419     void use(Edge edge)
2420     {
2421         ASSERT(edge->hasResult());
2422         if (!edge.doesKill())
2423             return;
2424         m_live.remove(edge.node());
2425     }
2426     
2427     // Wrapper used only for DFG_NODE_DO_TO_CHILDREN
2428     void use(Node*, Edge edge)
2429     {
2430         use(edge);
2431     }
2432     
2433     LBasicBlock lowBlock(BasicBlock* block)
2434     {
2435         return m_blocks.get(block);
2436     }
2437     
2438     void initializeOSRExitStateForBlock()
2439     {
2440         for (unsigned i = m_valueSources.size(); i--;) {
2441             FlushFormat format = m_highBlock->ssa->flushFormatAtHead[i];
2442             switch (format) {
2443             case DeadFlush: {
2444                 // Must consider available nodes instead.
2445                 Node* node = m_highBlock->ssa->availabilityAtHead[i];
2446                 if (!node) {
2447                     m_valueSources[i] = ValueSource(SourceIsDead);
2448                     break;
2449                 }
2450                 
2451                 m_valueSources[i] = ValueSource(node);
2452                 break;
2453             }
2454                 
2455             case FlushedInt32:
2456                 m_valueSources[i] = ValueSource(Int32InJSStack);
2457                 break;
2458                 
2459             case FlushedDouble:
2460                 m_valueSources[i] = ValueSource(DoubleInJSStack);
2461                 break;
2462                 
2463             case FlushedCell:
2464             case FlushedBoolean:
2465             case FlushedJSValue:
2466                 m_valueSources[i] = ValueSource(ValueInJSStack);
2467                 break;
2468             }
2469         }
2470     }
2471     
2472     void appendOSRExit(
2473         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2474         SpeculationDirection direction, FormattedValue recovery)
2475     {
2476         if (Options::ftlTrapsOnOSRExit()) {
2477             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2478             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2479             
2480             m_out.branch(failCondition, failCase, continuation);
2481             
2482             LBasicBlock lastNext = m_out.appendTo(failCase, continuation);
2483             m_out.trap();
2484             m_out.unreachable();
2485             
2486             m_out.appendTo(continuation, lastNext);
2487             return;
2488         }
2489         
2490         if (verboseCompilationEnabled())
2491             dataLog("    OSR exit with value sources: ", m_valueSources, "\n");
2492         
2493         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.osrExit.size());
2494         unsigned index = m_ftlState.osrExit.size();
2495         
2496         m_ftlState.jitCode->osrExit.append(OSRExit(
2497             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
2498             m_codeOriginForExitTarget, m_codeOriginForExitProfile, m_lastSetOperand,
2499             m_valueSources.numberOfArguments(), m_valueSources.numberOfLocals()));
2500         m_ftlState.osrExit.append(OSRExitCompilationInfo());
2501         
2502         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
2503         OSRExitCompilationInfo& info = m_ftlState.osrExit.last();
2504
2505         LBasicBlock lastNext = 0;
2506         LBasicBlock continuation = 0;
2507         
2508         if (!Options::useLLVMOSRExitIntrinsic()) {
2509             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2510             continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2511             
2512             m_out.branch(failCondition, failCase, continuation);
2513
2514             m_out.appendTo(m_prologue);
2515             info.m_thunkAddress = buildAlloca(m_out.m_builder, m_out.intPtr);
2516         
2517             lastNext = m_out.appendTo(failCase, continuation);
2518         }
2519         
2520         if (Options::ftlOSRExitOmitsMarshalling()) {
2521             m_out.call(
2522                 m_out.intToPtr(
2523                     m_out.get(info.m_thunkAddress),
2524                     pointerType(functionType(m_out.voidType))));
2525         } else
2526             emitOSRExitCall(failCondition, index, exit, info, lowValue, direction, recovery);
2527         
2528         if (!Options::useLLVMOSRExitIntrinsic()) {
2529             m_out.unreachable();
2530             
2531             m_out.appendTo(continuation, lastNext);
2532         
2533             m_exitThunkGenerator.emitThunk(index);
2534         }
2535     }
2536     
2537     void emitOSRExitCall(
2538         LValue failCondition, unsigned index, OSRExit& exit, OSRExitCompilationInfo& info,
2539         FormattedValue lowValue, SpeculationDirection direction, FormattedValue recovery)
2540     {
2541         ExitArgumentList arguments;
2542         
2543         if (Options::useLLVMOSRExitIntrinsic()) {
2544             arguments.append(failCondition);
2545             arguments.append(m_out.constInt32(index));
2546         }
2547         
2548         arguments.append(m_callFrame);
2549         if (!!lowValue)
2550             arguments.append(lowValue.value());
2551         
2552         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
2553             ValueSource source = m_valueSources[i];
2554             
2555             switch (source.kind()) {
2556             case ValueInJSStack:
2557                 exit.m_values[i] = ExitValue::inJSStack();
2558                 break;
2559             case Int32InJSStack:
2560                 exit.m_values[i] = ExitValue::inJSStackAsInt32();
2561                 break;
2562             case DoubleInJSStack:
2563                 exit.m_values[i] = ExitValue::inJSStackAsDouble();
2564                 break;
2565             case SourceIsDead:
2566                 exit.m_values[i] = ExitValue::dead();
2567                 break;
2568             case HaveNode:
2569                 addExitArgumentForNode(exit, arguments, i, source.node());
2570                 break;
2571             default:
2572                 RELEASE_ASSERT_NOT_REACHED();
2573                 break;
2574             }
2575         }
2576         
2577         if (verboseCompilationEnabled())
2578             dataLog("        Exit values: ", exit.m_values, "\n");
2579         
2580         if (direction == ForwardSpeculation) {
2581             ASSERT(m_node);
2582             exit.convertToForward(m_highBlock, m_node, m_nodeIndex, recovery, arguments);
2583         }
2584         
2585         // So, the really lame thing here is that we have to build an LLVM function type.
2586         // Booo.
2587         Vector<LType, 16> argumentTypes;
2588         for (unsigned i = 0; i < arguments.size(); ++i)
2589             argumentTypes.append(typeOf(arguments[i]));
2590         
2591         if (Options::useLLVMOSRExitIntrinsic()) {
2592             m_out.call(m_out.osrExitIntrinsic(), arguments);
2593             return;
2594         }
2595         
2596         m_out.call(
2597             m_out.intToPtr(
2598                 m_out.get(info.m_thunkAddress),
2599                 pointerType(functionType(m_out.voidType, argumentTypes))),
2600             arguments);
2601     }
2602     
2603     void addExitArgumentForNode(
2604         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
2605     {
2606         ASSERT(node->shouldGenerate());
2607         ASSERT(node->hasResult());
2608
2609         if (tryToSetConstantExitArgument(exit, index, node))
2610             return;
2611         
2612         if (!isLive(node)) {
2613             bool found = false;
2614             
2615             if (needsOSRBackwardRewiring(node->op())) {
2616                 node = node->child1().node();
2617                 if (tryToSetConstantExitArgument(exit, index, node))
2618                     return;
2619                 if (isLive(node))
2620                     found = true;
2621             }
2622             
2623             if (!found) {
2624                 Node* int32ToDouble = 0;
2625                 Node* valueToInt32 = 0;
2626                 Node* uint32ToNumber = 0;
2627                 Node* doubleAsInt32 = 0;
2628                 
2629                 HashSet<Node*>::iterator iter = m_live.begin();
2630                 HashSet<Node*>::iterator end = m_live.end();
2631                 for (; iter != end; ++iter) {
2632                     Node* candidate = *iter;
2633                     if (!candidate->child1())
2634                         continue;
2635                     if (candidate->child1() != node)
2636                         continue;
2637                     switch (candidate->op()) {
2638                     case Int32ToDouble:
2639                         int32ToDouble = candidate;
2640                         break;
2641                     case ValueToInt32:
2642                         valueToInt32 = candidate;
2643                         break;
2644                     case UInt32ToNumber:
2645                         uint32ToNumber = candidate;
2646                         break;
2647                     case DoubleAsInt32:
2648                         uint32ToNumber = candidate;
2649                         break;
2650                     default:
2651                         ASSERT(!needsOSRForwardRewiring(candidate->op()));
2652                         break;
2653                     }
2654                 }
2655                 
2656                 if (doubleAsInt32)
2657                     node = doubleAsInt32;
2658                 else if (int32ToDouble)
2659                     node = int32ToDouble;
2660                 else if (valueToInt32)
2661                     node = valueToInt32;
2662                 else if (uint32ToNumber)
2663                     node = uint32ToNumber;
2664                 
2665                 if (isLive(node))
2666                     found = true;
2667             }
2668             
2669             if (!found) {
2670                 exit.m_values[index] = ExitValue::dead();
2671                 return;
2672             }
2673         }
2674         
2675         ASSERT(isLive(node));
2676
2677         LoweredNodeValue value = m_int32Values.get(node);
2678         if (isValid(value)) {
2679             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
2680             return;
2681         }
2682         
2683         value = m_booleanValues.get(node);
2684         if (isValid(value)) {
2685             addExitArgument(exit, arguments, index, ValueFormatBoolean, value.value());
2686             return;
2687         }
2688         
2689         value = m_jsValueValues.get(node);
2690         if (isValid(value)) {
2691             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
2692             return;
2693         }
2694         
2695         value = m_doubleValues.get(node);
2696         if (isValid(value)) {
2697             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
2698             return;
2699         }
2700
2701         dataLog("Cannot find value for node: ", node, "\n");
2702         RELEASE_ASSERT_NOT_REACHED();
2703     }
2704     
2705     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
2706     {
2707         if (!node)
2708             return false;
2709         
2710         switch (node->op()) {
2711         case JSConstant:
2712         case WeakJSConstant:
2713             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
2714             return true;
2715         case PhantomArguments:
2716             // FIXME: implement PhantomArguments.
2717             // https://bugs.webkit.org/show_bug.cgi?id=113986
2718             RELEASE_ASSERT_NOT_REACHED();
2719             return true;
2720         default:
2721             return false;
2722         }
2723     }
2724     
2725     void addExitArgument(
2726         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
2727         LValue value)
2728     {
2729         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
2730         arguments.append(value);
2731     }
2732     
2733     void linkOSRExitsAndCompleteInitializationBlocks()
2734     {
2735         MacroAssemblerCodeRef osrExitThunk =
2736             vm().getCTIStub(osrExitGenerationThunkGenerator);
2737         CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
2738         
2739         m_out.appendTo(m_prologue);
2740         m_out.jump(m_initialization);
2741         
2742         m_out.appendTo(m_initialization);
2743         
2744         if (m_exitThunkGenerator.didThings()) {
2745             OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
2746                 vm(), &m_exitThunkGenerator, m_ftlState.graph.m_codeBlock,
2747                 JITCompilationMustSucceed));
2748         
2749             ASSERT(m_ftlState.osrExit.size() == m_ftlState.jitCode->osrExit.size());
2750         
2751             for (unsigned i = 0; i < m_ftlState.osrExit.size(); ++i) {
2752                 OSRExitCompilationInfo& info = m_ftlState.osrExit[i];
2753                 OSRExit& exit = m_ftlState.jitCode->osrExit[i];
2754             
2755                 linkBuffer->link(info.m_thunkJump, target);
2756             
2757                 m_out.set(
2758                     m_out.constIntPtr(
2759                         linkBuffer->locationOf(info.m_thunkLabel).executableAddress()),
2760                     info.m_thunkAddress);
2761             
2762                 exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
2763             }
2764         
2765             m_ftlState.finalizer->initializeExitThunksLinkBuffer(linkBuffer.release());
2766         }
2767
2768         m_out.jump(lowBlock(m_graph.block(0)));
2769     }
2770     
2771     void observeMovHint(Node* node)
2772     {
2773         ASSERT(node->containsMovHint());
2774         ASSERT(node->op() != ZombieHint);
2775         
2776         int operand = node->local();
2777         
2778         m_lastSetOperand = operand;
2779         m_valueSources.operand(operand) = ValueSource(node->child1().node());
2780     }
2781     
2782     void setInt32(Node* node, LValue value)
2783     {
2784         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
2785     }
2786     void setJSValue(Node* node, LValue value)
2787     {
2788         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
2789     }
2790     void setBoolean(Node* node, LValue value)
2791     {
2792         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
2793     }
2794     void setStorage(Node* node, LValue value)
2795     {
2796         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
2797     }
2798     void setDouble(Node* node, LValue value)
2799     {
2800         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
2801     }
2802
2803     void setInt32(LValue value)
2804     {
2805         setInt32(m_node, value);
2806     }
2807     void setJSValue(LValue value)
2808     {
2809         setJSValue(m_node, value);
2810     }
2811     void setBoolean(LValue value)
2812     {
2813         setBoolean(m_node, value);
2814     }
2815     void setStorage(LValue value)
2816     {
2817         setStorage(m_node, value);
2818     }
2819     void setDouble(LValue value)
2820     {
2821         setDouble(m_node, value);
2822     }
2823     
2824     bool isValid(const LoweredNodeValue& value)
2825     {
2826         if (!value)
2827             return false;
2828         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
2829             return false;
2830         return true;
2831     }
2832     
2833     void addWeakReference(JSCell* target)
2834     {
2835         m_graph.m_plan.weakReferences.addLazily(target);
2836     }
2837     
2838     LValue weakPointer(JSCell* pointer)
2839     {
2840         addWeakReference(pointer);
2841         return m_out.constIntPtr(pointer);
2842     }
2843     
2844     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
2845     {
2846         return m_out.address(base, m_heaps.variables[operand], offset);
2847     }
2848     TypedPointer payloadFor(LValue base, int operand)
2849     {
2850         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
2851     }
2852     TypedPointer tagFor(LValue base, int operand)
2853     {
2854         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
2855     }
2856     TypedPointer addressFor(int operand)
2857     {
2858         return addressFor(m_callFrame, operand);
2859     }
2860     TypedPointer payloadFor(int operand)
2861     {
2862         return payloadFor(m_callFrame, operand);
2863     }
2864     TypedPointer tagFor(int operand)
2865     {
2866         return tagFor(m_callFrame, operand);
2867     }
2868     
2869     VM& vm() { return m_graph.m_vm; }
2870     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
2871     
2872     Graph& m_graph;
2873     State& m_ftlState;
2874     AbstractHeapRepository m_heaps;
2875     Output m_out;
2876     
2877     LBasicBlock m_prologue;
2878     LBasicBlock m_initialization;
2879     HashMap<BasicBlock*, LBasicBlock> m_blocks;
2880     
2881     LValue m_callFrame;
2882     LValue m_tagTypeNumber;
2883     LValue m_tagMask;
2884     
2885     HashMap<Node*, LoweredNodeValue> m_int32Values;
2886     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
2887     HashMap<Node*, LoweredNodeValue> m_booleanValues;
2888     HashMap<Node*, LoweredNodeValue> m_storageValues;
2889     HashMap<Node*, LoweredNodeValue> m_doubleValues;
2890     HashSet<Node*> m_live;
2891     
2892     HashMap<Node*, LValue> m_phis;
2893     
2894     Operands<ValueSource> m_valueSources;
2895     int m_lastSetOperand;
2896     ExitThunkGenerator m_exitThunkGenerator;
2897     
2898     InPlaceAbstractState m_state;
2899     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
2900     BasicBlock* m_highBlock;
2901     BasicBlock* m_nextHighBlock;
2902     LBasicBlock m_nextLowBlock;
2903     
2904     CodeOrigin m_codeOriginForExitTarget;
2905     CodeOrigin m_codeOriginForExitProfile;
2906     unsigned m_nodeIndex;
2907     Node* m_node;
2908     SpeculationDirection m_direction;
2909 };
2910
2911 void lowerDFGToLLVM(State& state)
2912 {
2913     LowerDFGToLLVM lowering(state);
2914     lowering.lower();
2915 }
2916
2917 } } // namespace JSC::FTL
2918
2919 #endif // ENABLE(FTL_JIT)
2920