2df2bc6b44d9ad0188c7ee79814e79ce37dfddef
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToLLVM.cpp
1 /*
2  * Copyright (C) 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToLLVM.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "CodeBlockWithJITType.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGInPlaceAbstractState.h"
34 #include "FTLAbstractHeapRepository.h"
35 #include "FTLExitThunkGenerator.h"
36 #include "FTLFormattedValue.h"
37 #include "FTLLoweredNodeValue.h"
38 #include "FTLOutput.h"
39 #include "FTLThunks.h"
40 #include "FTLValueSource.h"
41 #include "LinkBuffer.h"
42 #include "Operations.h"
43 #include <wtf/ProcessID.h>
44
45 namespace JSC { namespace FTL {
46
47 using namespace DFG;
48
49 static int compileCounter;
50
51 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
52 // significantly less dead code.
53 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
54         FormattedValue _ftc_lowValue = (lowValue);                      \
55         Edge _ftc_highValue = (highValue);                              \
56         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
57         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
58             break;                                                      \
59         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
60     } while (false)
61
62 class LowerDFGToLLVM {
63 public:
64     LowerDFGToLLVM(State& state)
65         : m_graph(state.graph)
66         , m_ftlState(state)
67         , m_heaps(state.context)
68         , m_out(state.context)
69         , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
70         , m_lastSetOperand(std::numeric_limits<int>::max())
71         , m_exitThunkGenerator(state)
72         , m_state(state.graph)
73         , m_interpreter(state.graph, m_state)
74     {
75     }
76     
77     void lower()
78     {
79         CString name;
80         if (verboseCompilationEnabled()) {
81             name = toCString(
82                 "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
83                 "_", codeBlock()->hash());
84         } else
85             name = "jsBody";
86         
87         m_graph.m_dominators.computeIfNecessary(m_graph);
88         
89         m_ftlState.module =
90             LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
91         
92         m_ftlState.function = addFunction(
93             m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
94         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
95         
96         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
97         
98         m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
99         m_out.appendTo(m_prologue);
100         createPhiVariables();
101         
102         m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
103
104         m_callFrame = m_out.param(0);
105         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
106         m_tagMask = m_out.constInt64(TagMask);
107         
108         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
109             m_highBlock = m_graph.block(blockIndex);
110             if (!m_highBlock)
111                 continue;
112             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
113         }
114         
115         Vector<BasicBlock*> depthFirst;
116         m_graph.getBlocksInDepthFirstOrder(depthFirst);
117         for (unsigned i = 0; i < depthFirst.size(); ++i)
118             compileBlock(depthFirst[i]);
119         
120         // And now complete the initialization block.
121         linkOSRExitsAndCompleteInitializationBlocks();
122
123         if (Options::dumpLLVMIR())
124             dumpModule(m_ftlState.module);
125         
126         if (verboseCompilationEnabled())
127             m_ftlState.dumpState("after lowering");
128         if (validationEnabled())
129             verifyModule(m_ftlState.module);
130     }
131
132 private:
133     
134     void createPhiVariables()
135     {
136         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
137             BasicBlock* block = m_graph.block(blockIndex);
138             if (!block)
139                 continue;
140             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
141                 Node* node = block->at(nodeIndex);
142                 if (node->op() != Phi)
143                     continue;
144                 LType type;
145                 switch (node->flags() & NodeResultMask) {
146                 case NodeResultNumber:
147                     type = m_out.doubleType;
148                     break;
149                 case NodeResultInt32:
150                     type = m_out.int32;
151                     break;
152                 case NodeResultBoolean:
153                     type = m_out.boolean;
154                     break;
155                 case NodeResultJS:
156                     type = m_out.int64;
157                     break;
158                 default:
159                     RELEASE_ASSERT_NOT_REACHED();
160                     break;
161                 }
162                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
163             }
164         }
165     }
166     
167     void compileBlock(BasicBlock* block)
168     {
169         if (!block)
170             return;
171         
172         if (verboseCompilationEnabled())
173             dataLog("Compiling block ", *block, "\n");
174         
175         m_highBlock = block;
176         
177         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
178         
179         m_nextHighBlock = 0;
180         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
181             m_nextHighBlock = m_graph.block(nextBlockIndex);
182             if (m_nextHighBlock)
183                 break;
184         }
185         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
186         
187         // All of this effort to find the next block gives us the ability to keep the
188         // generated IR in roughly program order. This ought not affect the performance
189         // of the generated code (since we expect LLVM to reorder things) but it will
190         // make IR dumps easier to read.
191         m_out.appendTo(lowBlock, m_nextLowBlock);
192         
193         if (!m_highBlock->cfaHasVisited) {
194             m_out.crash();
195             return;
196         }
197         
198         initializeOSRExitStateForBlock();
199         
200         m_live = block->ssa->liveAtHead;
201         
202         m_state.reset();
203         m_state.beginBasicBlock(m_highBlock);
204         
205         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
206             if (!compileNode(m_nodeIndex))
207                 break;
208         }
209     }
210     
211     bool compileNode(unsigned nodeIndex)
212     {
213         if (!m_state.isValid()) {
214             m_out.unreachable();
215             return false;
216         }
217         
218         m_node = m_highBlock->at(nodeIndex);
219         m_codeOriginForExitProfile = m_node->codeOrigin;
220         m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
221         
222         if (verboseCompilationEnabled())
223             dataLog("Lowering ", m_node, "\n");
224         
225         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
226         
227         m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
228         
229         switch (m_node->op()) {
230         case Upsilon:
231             compileUpsilon();
232             break;
233         case Phi:
234             compilePhi();
235             break;
236         case JSConstant:
237             compileJSConstant();
238             break;
239         case WeakJSConstant:
240             compileWeakJSConstant();
241             break;
242         case GetArgument:
243             compileGetArgument();
244             break;
245         case GetLocal:
246             compileGetLocal();
247             break;
248         case SetLocal:
249             compileSetLocal();
250             break;
251         case MovHint:
252             compileMovHint();
253             break;
254         case ZombieHint:
255             compileZombieHint();
256             break;
257         case MovHintAndCheck:
258             compileMovHintAndCheck();
259             break;
260         case Phantom:
261             compilePhantom();
262             break;
263         case Flush:
264         case PhantomLocal:
265         case SetArgument:
266             break;
267         case ArithAdd:
268         case ValueAdd:
269             compileAdd();
270             break;
271         case ArithSub:
272             compileArithSub();
273             break;
274         case ArithMul:
275             compileArithMul();
276             break;
277         case ArithDiv:
278             compileArithDiv();
279             break;
280         case ArithMod:
281             compileArithMod();
282             break;
283         case ArithMin:
284         case ArithMax:
285             compileArithMinOrMax();
286             break;
287         case ArithAbs:
288             compileArithAbs();
289             break;
290         case ArithNegate:
291             compileArithNegate();
292             break;
293         case BitAnd:
294             compileBitAnd();
295             break;
296         case BitOr:
297             compileBitOr();
298             break;
299         case BitXor:
300             compileBitXor();
301             break;
302         case BitRShift:
303             compileBitRShift();
304             break;
305         case BitLShift:
306             compileBitLShift();
307             break;
308         case BitURShift:
309             compileBitURShift();
310             break;
311         case UInt32ToNumber:
312             compileUInt32ToNumber();
313             break;
314         case Int32ToDouble:
315             compileInt32ToDouble();
316             break;
317         case CheckStructure:
318             compileCheckStructure();
319             break;
320         case StructureTransitionWatchpoint:
321             compileStructureTransitionWatchpoint();
322             break;
323         case ArrayifyToStructure:
324             compileArrayifyToStructure();
325             break;
326         case PutStructure:
327             compilePutStructure();
328             break;
329         case PhantomPutStructure:
330             compilePhantomPutStructure();
331             break;
332         case GetButterfly:
333             compileGetButterfly();
334             break;
335         case GetArrayLength:
336             compileGetArrayLength();
337             break;
338         case GetByVal:
339             compileGetByVal();
340             break;
341         case PutByVal:
342         case PutByValAlias:
343             compilePutByVal();
344             break;
345         case GetByOffset:
346             compileGetByOffset();
347             break;
348         case PutByOffset:
349             compilePutByOffset();
350             break;
351         case GetGlobalVar:
352             compileGetGlobalVar();
353             break;
354         case PutGlobalVar:
355             compilePutGlobalVar();
356             break;
357         case CompareEq:
358             compileCompareEq();
359             break;
360         case CompareEqConstant:
361             compileCompareEqConstant();
362             break;
363         case CompareStrictEq:
364             compileCompareStrictEq();
365             break;
366         case CompareStrictEqConstant:
367             compileCompareStrictEqConstant();
368             break;
369         case CompareLess:
370             compileCompareLess();
371             break;
372         case CompareLessEq:
373             compileCompareLessEq();
374             break;
375         case CompareGreater:
376             compileCompareGreater();
377             break;
378         case CompareGreaterEq:
379             compileCompareGreaterEq();
380             break;
381         case LogicalNot:
382             compileLogicalNot();
383             break;
384         case Jump:
385             compileJump();
386             break;
387         case Branch:
388             compileBranch();
389             break;
390         case Switch:
391             compileSwitch();
392             break;
393         case Return:
394             compileReturn();
395             break;
396         case ForceOSRExit:
397             compileForceOSRExit();
398             break;
399         default:
400             RELEASE_ASSERT_NOT_REACHED();
401             break;
402         }
403         
404         if (m_node->shouldGenerate())
405             DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
406         
407         if (m_node->adjustedRefCount())
408             m_live.add(m_node);
409         
410         if (shouldExecuteEffects)
411             m_interpreter.executeEffects(nodeIndex);
412         
413         return true;
414     }
415     
416     void compileUpsilon()
417     {
418         LValue destination = m_phis.get(m_node->phi());
419         
420         switch (m_node->child1().useKind()) {
421         case NumberUse:
422             m_out.set(lowDouble(m_node->child1()), destination);
423             break;
424         case Int32Use:
425             m_out.set(lowInt32(m_node->child1()), destination);
426             break;
427         case BooleanUse:
428             m_out.set(lowBoolean(m_node->child1()), destination);
429             break;
430         case CellUse:
431             m_out.set(lowCell(m_node->child1()), destination);
432             break;
433         case UntypedUse:
434             m_out.set(lowJSValue(m_node->child1()), destination);
435             break;
436         default:
437             RELEASE_ASSERT_NOT_REACHED();
438             break;
439         }
440     }
441     
442     void compilePhi()
443     {
444         LValue source = m_phis.get(m_node);
445         
446         switch (m_node->flags() & NodeResultMask) {
447         case NodeResultNumber:
448             setDouble(m_out.get(source));
449             break;
450         case NodeResultInt32:
451             setInt32(m_out.get(source));
452             break;
453         case NodeResultBoolean:
454             setBoolean(m_out.get(source));
455             break;
456         case NodeResultJS:
457             setJSValue(m_out.get(source));
458             break;
459         default:
460             RELEASE_ASSERT_NOT_REACHED();
461             break;
462         }
463     }
464     
465     void compileJSConstant()
466     {
467         JSValue value = m_graph.valueOfJSConstant(m_node);
468         if (value.isDouble())
469             setDouble(m_out.constDouble(value.asDouble()));
470         else
471             setJSValue(m_out.constInt64(JSValue::encode(value)));
472     }
473     
474     void compileWeakJSConstant()
475     {
476         setJSValue(weakPointer(m_node->weakConstant()));
477     }
478     
479     void compileGetArgument()
480     {
481         VariableAccessData* variable = m_node->variableAccessData();
482         int operand = variable->operand();
483
484         LValue jsValue = m_out.load64(addressFor(operand));
485
486         switch (useKindFor(variable->flushFormat())) {
487         case Int32Use:
488             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
489             setInt32(unboxInt32(jsValue));
490             break;
491         case CellUse:
492             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
493             setJSValue(jsValue);
494             break;
495         case BooleanUse:
496             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
497             setBoolean(unboxBoolean(jsValue));
498             break;
499         case UntypedUse:
500             setJSValue(jsValue);
501             break;
502         default:
503             RELEASE_ASSERT_NOT_REACHED();
504             break;
505         }
506     }
507     
508     void compileGetLocal()
509     {
510         // GetLocals arise only for captured variables.
511         
512         VariableAccessData* variable = m_node->variableAccessData();
513         AbstractValue& value = m_state.variables().operand(variable->local());
514         
515         RELEASE_ASSERT(variable->isCaptured());
516         
517         if (isInt32Speculation(value.m_value))
518             setInt32(m_out.load32(payloadFor(variable->local())));
519         else
520             setJSValue(m_out.load64(addressFor(variable->local())));
521     }
522     
523     void compileSetLocal()
524     {
525         observeMovHint(m_node);
526         
527         VariableAccessData* variable = m_node->variableAccessData();
528         SpeculatedType prediction = variable->argumentAwarePrediction();
529         
530         if (variable->shouldUnboxIfPossible()) {
531             if (variable->shouldUseDoubleFormat()) {
532                 LValue value = lowDouble(m_node->child1());
533                 m_out.storeDouble(value, addressFor(variable->local()));
534                 m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
535                 return;
536             }
537             
538             if (isInt32Speculation(prediction)) {
539                 LValue value = lowInt32(m_node->child1());
540                 m_out.store32(value, payloadFor(variable->local()));
541                 m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
542                 return;
543             }
544             if (isCellSpeculation(prediction)) {
545                 LValue value = lowCell(m_node->child1());
546                 m_out.store64(value, addressFor(variable->local()));
547                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
548                 return;
549             }
550             if (isBooleanSpeculation(prediction)) {
551                 speculateBoolean(m_node->child1());
552                 m_out.store64(
553                     lowJSValue(m_node->child1(), ManualOperandSpeculation),
554                     addressFor(variable->local()));
555                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
556                 return;
557             }
558         }
559         
560         LValue value = lowJSValue(m_node->child1());
561         m_out.store64(value, addressFor(variable->local()));
562         m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
563     }
564     
565     void compileMovHint()
566     {
567         observeMovHint(m_node);
568     }
569     
570     void compileZombieHint()
571     {
572         VariableAccessData* data = m_node->variableAccessData();
573         m_lastSetOperand = data->local();
574         m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
575     }
576     
577     void compileMovHintAndCheck()
578     {
579         observeMovHint(m_node);
580         speculate(m_node->child1());
581     }
582     
583     void compilePhantom()
584     {
585         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
586     }
587     
588     void compileAdd()
589     {
590         switch (m_node->binaryUseKind()) {
591         case Int32Use: {
592             LValue left = lowInt32(m_node->child1());
593             LValue right = lowInt32(m_node->child2());
594             
595             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
596                 setInt32(m_out.add(left, right));
597                 break;
598             }
599             
600             LValue result = m_out.addWithOverflow32(left, right);
601             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
602             setInt32(m_out.extractValue(result, 0));
603             break;
604         }
605             
606         case NumberUse: {
607             setDouble(
608                 m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
609             break;
610         }
611             
612         default:
613             RELEASE_ASSERT_NOT_REACHED();
614             break;
615         }
616     }
617     
618     void compileArithSub()
619     {
620         switch (m_node->binaryUseKind()) {
621         case Int32Use: {
622             LValue left = lowInt32(m_node->child1());
623             LValue right = lowInt32(m_node->child2());
624             
625             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
626                 setInt32(m_out.sub(left, right));
627                 break;
628             }
629             
630             LValue result = m_out.subWithOverflow32(left, right);
631             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
632             setInt32(m_out.extractValue(result, 0));
633             break;
634         }
635             
636         case NumberUse: {
637             setDouble(
638                 m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
639             break;
640         }
641             
642         default:
643             RELEASE_ASSERT_NOT_REACHED();
644             break;
645         }
646     }
647     
648     void compileArithMul()
649     {
650         switch (m_node->binaryUseKind()) {
651         case Int32Use: {
652             LValue left = lowInt32(m_node->child1());
653             LValue right = lowInt32(m_node->child2());
654             
655             LValue result;
656             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
657                 result = m_out.mul(left, right);
658             else {
659                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
660                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
661                 result = m_out.extractValue(overflowResult, 0);
662             }
663             
664             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
665                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
666                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
667                 
668                 m_out.branch(m_out.notZero32(result), continuation, slowCase);
669                 
670                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
671                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
672                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
673                 m_out.jump(continuation);
674                 m_out.appendTo(continuation, lastNext);
675             }
676             
677             setInt32(result);
678             break;
679         }
680             
681         case NumberUse: {
682             setDouble(
683                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
684             break;
685         }
686             
687         default:
688             RELEASE_ASSERT_NOT_REACHED();
689             break;
690         }
691     }
692     
693     void compileArithDiv()
694     {
695         switch (m_node->binaryUseKind()) {
696         case Int32Use: {
697             LValue numerator = lowInt32(m_node->child1());
698             LValue denominator = lowInt32(m_node->child2());
699             
700             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
701             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
702             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
703             
704             Vector<ValueFromBlock, 3> results;
705             
706             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
707             
708             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
709             
710             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
711             
712             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
713             
714             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
715                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
716                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
717                 m_out.jump(continuation);
718             } else {
719                 // This is the case where we convert the result to an int after we're done. So,
720                 // if the denominator is zero, then the result should be result should be zero.
721                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
722                 // check above) and the numerator is -2^31 then the result should be -2^31.
723                 
724                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
725                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
726                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
727                 
728                 m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
729                 
730                 m_out.appendTo(divByZero, notDivByZero);
731                 results.append(m_out.anchor(m_out.int32Zero));
732                 m_out.jump(done);
733                 
734                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
735                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
736                 
737                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
738                 results.append(m_out.anchor(neg2ToThe31));
739                 m_out.jump(done);
740             }
741             
742             m_out.appendTo(continuation, done);
743             
744             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
745                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
746                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
747                 
748                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
749                 
750                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
751                 
752                 speculate(
753                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
754                 
755                 m_out.jump(numeratorContinuation);
756                 
757                 m_out.appendTo(numeratorContinuation, innerLastNext);
758             }
759             
760             LValue divisionResult = m_out.div(numerator, denominator);
761             
762             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
763                 speculate(
764                     Overflow, noValue(), 0,
765                     m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
766             }
767             
768             results.append(m_out.anchor(divisionResult));
769             m_out.jump(done);
770             
771             m_out.appendTo(done, lastNext);
772             
773             setInt32(m_out.phi(m_out.int32, results));
774             break;
775         }
776             
777         case NumberUse: {
778             setDouble(
779                 m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
780             break;
781         }
782             
783         default:
784             RELEASE_ASSERT_NOT_REACHED();
785             break;
786         }
787     }
788     
789     void compileArithMod()
790     {
791         switch (m_node->binaryUseKind()) {
792         case Int32Use: {
793             LValue numerator = lowInt32(m_node->child1());
794             LValue denominator = lowInt32(m_node->child2());
795             
796             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
797             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
798             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
799             
800             Vector<ValueFromBlock, 3> results;
801             
802             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
803             
804             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
805             
806             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
807             
808             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
809             
810             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
811             // separate case for that. But it probably doesn't matter so much.
812             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
813                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
814                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
815                 m_out.jump(continuation);
816             } else {
817                 // This is the case where we convert the result to an int after we're done. So,
818                 // if the denominator is zero, then the result should be result should be zero.
819                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
820                 // check above) and the numerator is -2^31 then the result should be -2^31.
821                 
822                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
823                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
824                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
825                 
826                 m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
827                 
828                 m_out.appendTo(modByZero, notModByZero);
829                 results.append(m_out.anchor(m_out.int32Zero));
830                 m_out.jump(done);
831                 
832                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
833                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
834                 
835                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
836                 results.append(m_out.anchor(m_out.int32Zero));
837                 m_out.jump(done);
838             }
839             
840             m_out.appendTo(continuation, done);
841             
842             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
843                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod zero numerator"));
844                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
845                 
846                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
847                 
848                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
849                 
850                 speculate(
851                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
852                 
853                 m_out.jump(numeratorContinuation);
854                 
855                 m_out.appendTo(numeratorContinuation, innerLastNext);
856             }
857             
858             results.append(m_out.anchor(m_out.rem(numerator, denominator)));
859             m_out.jump(done);
860             
861             m_out.appendTo(done, lastNext);
862             
863             setInt32(m_out.phi(m_out.int32, results));
864             break;
865         }
866             
867         case NumberUse: {
868             setDouble(
869                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
870             break;
871         }
872             
873         default:
874             RELEASE_ASSERT_NOT_REACHED();
875             break;
876         }
877     }
878
879     void compileArithMinOrMax()
880     {
881         switch (m_node->binaryUseKind()) {
882         case Int32Use: {
883             LValue left = lowInt32(m_node->child1());
884             LValue right = lowInt32(m_node->child2());
885             
886             setInt32(
887                 m_out.select(
888                     m_node->op() == ArithMin
889                         ? m_out.lessThan(left, right)
890                         : m_out.lessThan(right, left),
891                     left, right));
892             break;
893         }
894             
895         case NumberUse: {
896             LValue left = lowDouble(m_node->child1());
897             LValue right = lowDouble(m_node->child2());
898             
899             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
900             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
901             
902             Vector<ValueFromBlock, 2> results;
903             
904             results.append(m_out.anchor(left));
905             m_out.branch(
906                 m_node->op() == ArithMin
907                     ? m_out.doubleLessThan(left, right)
908                     : m_out.doubleGreaterThan(left, right),
909                 continuation, notLessThan);
910             
911             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
912             results.append(m_out.anchor(m_out.select(
913                 m_node->op() == ArithMin
914                     ? m_out.doubleGreaterThanOrEqual(left, right)
915                     : m_out.doubleLessThanOrEqual(left, right),
916                 right, m_out.constDouble(0.0 / 0.0))));
917             m_out.jump(continuation);
918             
919             m_out.appendTo(continuation, lastNext);
920             setDouble(m_out.phi(m_out.doubleType, results));
921             break;
922         }
923             
924         default:
925             RELEASE_ASSERT_NOT_REACHED();
926             break;
927         }
928     }
929     
930     void compileArithAbs()
931     {
932         switch (m_node->child1().useKind()) {
933         case Int32Use: {
934             LValue value = lowInt32(m_node->child1());
935             
936             LValue mask = m_out.aShr(value, m_out.constInt32(31));
937             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
938             
939             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
940             
941             setInt32(result);
942             break;
943         }
944             
945         case NumberUse: {
946             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
947             break;
948         }
949             
950         default:
951             RELEASE_ASSERT_NOT_REACHED();
952             break;
953         }
954     }
955     
956     void compileArithNegate()
957     {
958         switch (m_node->child1().useKind()) {
959         case Int32Use: {
960             LValue value = lowInt32(m_node->child1());
961             
962             LValue result;
963             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
964                 result = m_out.neg(value);
965             else {
966                 // We don't have a negate-with-overflow intrinsic. Hopefully this
967                 // does the trick, though.
968                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
969                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
970                 result = m_out.extractValue(overflowResult, 0);
971             }
972             
973             setInt32(result);
974             break;
975         }
976             
977         case NumberUse: {
978             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
979             break;
980         }
981             
982         default:
983             RELEASE_ASSERT_NOT_REACHED();
984             break;
985         }
986     }
987     
988     void compileBitAnd()
989     {
990         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
991     }
992     
993     void compileBitOr()
994     {
995         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
996     }
997     
998     void compileBitXor()
999     {
1000         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1001     }
1002     
1003     void compileBitRShift()
1004     {
1005         setInt32(m_out.aShr(
1006             lowInt32(m_node->child1()),
1007             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1008     }
1009     
1010     void compileBitLShift()
1011     {
1012         setInt32(m_out.shl(
1013             lowInt32(m_node->child1()),
1014             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1015     }
1016     
1017     void compileBitURShift()
1018     {
1019         setInt32(m_out.lShr(
1020             lowInt32(m_node->child1()),
1021             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1022     }
1023     
1024     void compileUInt32ToNumber()
1025     {
1026         LValue value = lowInt32(m_node->child1());
1027
1028         if (!nodeCanSpeculateInteger(m_node->arithNodeFlags())) {
1029             setDouble(m_out.unsignedToDouble(value));
1030             return;
1031         }
1032         
1033         speculateForward(
1034             Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
1035             FormattedValue(ValueFormatUInt32, value));
1036         setInt32(value);
1037     }
1038     
1039     void compileInt32ToDouble()
1040     {
1041         // This node is tricky to compile in the DFG backend because it tries to
1042         // avoid converting child1 to a double in-place, as that would make subsequent
1043         // int uses of of child1 fail. But the FTL needs no such special magic, since
1044         // unlike the DFG backend, the FTL allows each node to have multiple
1045         // contemporaneous low-level representations. So, this gives child1 a double
1046         // representation and then forwards that representation to m_node.
1047         
1048         setDouble(lowDouble(m_node->child1()));
1049     }
1050     
1051     void compileCheckStructure()
1052     {
1053         LValue cell = lowCell(m_node->child1());
1054         
1055         ExitKind exitKind;
1056         if (m_node->child1()->op() == WeakJSConstant)
1057             exitKind = BadWeakConstantCache;
1058         else
1059             exitKind = BadCache;
1060         
1061         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1062         
1063         if (m_node->structureSet().size() == 1) {
1064             speculate(
1065                 exitKind, jsValueValue(cell), 0,
1066                 m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
1067             return;
1068         }
1069         
1070         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1071         
1072         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1073         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1074             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1075             m_out.branch(
1076                 m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
1077                 continuation, nextStructure);
1078             m_out.appendTo(nextStructure);
1079         }
1080         
1081         speculate(
1082             exitKind, jsValueValue(cell), 0,
1083             m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
1084         
1085         m_out.jump(continuation);
1086         m_out.appendTo(continuation, lastNext);
1087     }
1088     
1089     void compileStructureTransitionWatchpoint()
1090     {
1091         addWeakReference(m_node->structure());
1092         
1093         // FIXME: Implement structure transition watchpoints.
1094         // https://bugs.webkit.org/show_bug.cgi?id=113647
1095         
1096         speculateCell(m_node->child1());
1097     }
1098     
1099     void compileArrayifyToStructure()
1100     {
1101         LValue cell = lowCell(m_node->child1());
1102         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1103         
1104         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1105         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1106         
1107         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1108         
1109         m_out.branch(
1110             m_out.notEqual(structure, weakPointer(m_node->structure())),
1111             unexpectedStructure, continuation);
1112         
1113         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1114         
1115         if (property) {
1116             switch (m_node->arrayMode().type()) {
1117             case Array::Int32:
1118             case Array::Double:
1119             case Array::Contiguous:
1120                 speculate(
1121                     Uncountable, noValue(), 0,
1122                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1123                 break;
1124             default:
1125                 break;
1126             }
1127         }
1128         
1129         switch (m_node->arrayMode().type()) {
1130         case Array::Int32:
1131             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1132             break;
1133         case Array::Double:
1134             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1135             break;
1136         case Array::Contiguous:
1137             if (m_node->arrayMode().conversion() == Array::RageConvert)
1138                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1139             else
1140                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1141             break;
1142         case Array::ArrayStorage:
1143         case Array::SlowPutArrayStorage:
1144             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1145             break;
1146         default:
1147             RELEASE_ASSERT_NOT_REACHED();
1148             break;
1149         }
1150         
1151         structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1152         speculate(
1153             BadIndexingType, jsValueValue(cell), 0,
1154             m_out.notEqual(structure, weakPointer(m_node->structure())));
1155         m_out.jump(continuation);
1156         
1157         m_out.appendTo(continuation, lastNext);
1158     }
1159     
1160     void compilePutStructure()
1161     {
1162         m_ftlState.jitCode->common.notifyCompilingStructureTransition(codeBlock(), m_node);
1163         
1164         m_out.store64(
1165             m_out.constIntPtr(m_node->structureTransitionData().newStructure),
1166             lowCell(m_node->child1()), m_heaps.JSCell_structure);
1167     }
1168     
1169     void compilePhantomPutStructure()
1170     {
1171         m_ftlState.jitCode->common.notifyCompilingStructureTransition(codeBlock(), m_node);
1172     }
1173     
1174     void compileGetButterfly()
1175     {
1176         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1177     }
1178     
1179     void compileGetArrayLength()
1180     {
1181         switch (m_node->arrayMode().type()) {
1182         case Array::Int32:
1183         case Array::Double:
1184         case Array::Contiguous: {
1185             setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1186             break;
1187         }
1188             
1189         default:
1190             RELEASE_ASSERT_NOT_REACHED();
1191             break;
1192         }
1193     }
1194     
1195     void compileGetByVal()
1196     {
1197         LValue index = lowInt32(m_node->child2());
1198         LValue storage = lowStorage(m_node->child3());
1199         
1200         switch (m_node->arrayMode().type()) {
1201         case Array::Int32:
1202         case Array::Contiguous: {
1203             if (m_node->arrayMode().isInBounds()) {
1204                 speculate(
1205                     OutOfBounds, noValue(), 0,
1206                     m_out.aboveOrEqual(
1207                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1208                 
1209                 LValue result = m_out.load64(m_out.baseIndex(
1210                     m_node->arrayMode().type() == Array::Int32 ?
1211                         m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1212                     storage, m_out.zeroExt(index, m_out.intPtr),
1213                     m_state.forNode(m_node->child2()).m_value));
1214                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1215                 setJSValue(result);
1216                 return;
1217             }
1218             
1219             // FIXME: Implement hole/OOB loads in the FTL.
1220             // https://bugs.webkit.org/show_bug.cgi?id=118077
1221             RELEASE_ASSERT_NOT_REACHED();
1222             return;
1223         }
1224             
1225         case Array::Double: {
1226             if (m_node->arrayMode().isInBounds()) {
1227                 if (m_node->arrayMode().isSaneChain()) {
1228                     // FIXME: Implement structure transition watchpoints.
1229                     // https://bugs.webkit.org/show_bug.cgi?id=113647
1230                 }
1231             
1232                 speculate(
1233                     OutOfBounds, noValue(), 0,
1234                     m_out.aboveOrEqual(
1235                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1236                 
1237                 LValue result = m_out.loadDouble(m_out.baseIndex(
1238                     m_heaps.indexedDoubleProperties,
1239                     storage, m_out.zeroExt(index, m_out.intPtr),
1240                     m_state.forNode(m_node->child2()).m_value));
1241                 
1242                 if (!m_node->arrayMode().isSaneChain()) {
1243                     speculate(
1244                         LoadFromHole, noValue(), 0,
1245                         m_out.doubleNotEqualOrUnordered(result, result));
1246                 }
1247                 setDouble(result);
1248                 break;
1249             }
1250             
1251             // FIXME: Implement hole/OOB loads in the FTL.
1252             // https://bugs.webkit.org/show_bug.cgi?id=118077
1253             RELEASE_ASSERT_NOT_REACHED();
1254             return;
1255         }
1256             
1257         default:
1258             RELEASE_ASSERT_NOT_REACHED();
1259             return;
1260         }
1261     }
1262     
1263     void compilePutByVal()
1264     {
1265         Edge child1 = m_graph.varArgChild(m_node, 0);
1266         Edge child2 = m_graph.varArgChild(m_node, 1);
1267         Edge child3 = m_graph.varArgChild(m_node, 2);
1268         Edge child4 = m_graph.varArgChild(m_node, 3);
1269
1270         LValue base = lowCell(child1);
1271         LValue index = lowInt32(child2);
1272         LValue storage = lowStorage(child4);
1273         
1274         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
1275         LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
1276             
1277         switch (m_node->arrayMode().type()) {
1278         case Array::Int32:
1279         case Array::Contiguous: {
1280             LValue value = lowJSValue(child3, ManualOperandSpeculation);
1281             
1282             if (m_node->arrayMode().type() == Array::Int32)
1283                 FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
1284             
1285             TypedPointer elementPointer = m_out.baseIndex(
1286                 m_node->arrayMode().type() == Array::Int32 ?
1287                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1288                 storage, m_out.zeroExt(index, m_out.intPtr),
1289                 m_state.forNode(child2).m_value);
1290             
1291             if (m_node->op() == PutByValAlias) {
1292                 m_out.store64(value, elementPointer);
1293                 break;
1294             }
1295             
1296             contiguousPutByValOutOfBounds(
1297                 codeBlock()->isStrictMode()
1298                     ? operationPutByValBeyondArrayBoundsStrict
1299                     : operationPutByValBeyondArrayBoundsNonStrict,
1300                 base, storage, index, value, continuation);
1301             
1302             m_out.store64(value, elementPointer);
1303             break;
1304         }
1305             
1306         case Array::Double: {
1307             LValue value = lowDouble(child3);
1308             
1309             FTL_TYPE_CHECK(
1310                 doubleValue(value), child3, SpecRealNumber,
1311                 m_out.doubleNotEqualOrUnordered(value, value));
1312             
1313             TypedPointer elementPointer = m_out.baseIndex(
1314                 m_heaps.indexedDoubleProperties,
1315                 storage, m_out.zeroExt(index, m_out.intPtr),
1316                 m_state.forNode(child2).m_value);
1317
1318             if (m_node->op() == PutByValAlias) {
1319                 m_out.storeDouble(value, elementPointer);
1320                 break;
1321             }
1322             
1323             contiguousPutByValOutOfBounds(
1324                 codeBlock()->isStrictMode()
1325                     ? operationPutDoubleByValBeyondArrayBoundsStrict
1326                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1327                 base, storage, index, value, continuation);
1328             
1329             m_out.storeDouble(value, elementPointer);
1330             break;
1331         }
1332             
1333         default:
1334             RELEASE_ASSERT_NOT_REACHED();
1335             break;
1336         }
1337
1338         m_out.jump(continuation);
1339         m_out.appendTo(continuation, outerLastNext);
1340     }
1341     
1342     void compileGetByOffset()
1343     {
1344         StorageAccessData& data =
1345             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1346         
1347         setJSValue(
1348             m_out.load64(
1349                 m_out.address(
1350                     m_heaps.properties[data.identifierNumber],
1351                     lowStorage(m_node->child1()),
1352                     offsetRelativeToBase(data.offset))));
1353     }
1354     
1355     void compilePutByOffset()
1356     {
1357         StorageAccessData& data =
1358             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1359         
1360         m_out.store64(
1361             lowJSValue(m_node->child3()),
1362             m_out.address(
1363                 m_heaps.properties[data.identifierNumber],
1364                 lowStorage(m_node->child1()),
1365                 offsetRelativeToBase(data.offset)));
1366     }
1367     
1368     void compileGetGlobalVar()
1369     {
1370         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
1371     }
1372     
1373     void compilePutGlobalVar()
1374     {
1375         m_out.store64(
1376             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
1377     }
1378     
1379     void compileCompareEq()
1380     {
1381         if (m_node->isBinaryUseKind(Int32Use)
1382             || m_node->isBinaryUseKind(NumberUse)
1383             || m_node->isBinaryUseKind(ObjectUse)) {
1384             compileCompareStrictEq();
1385             return;
1386         }
1387         
1388         RELEASE_ASSERT_NOT_REACHED();
1389     }
1390     
1391     void compileCompareEqConstant()
1392     {
1393         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
1394         masqueradesAsUndefinedWatchpointIfIsStillValid();
1395         setBoolean(
1396             equalNullOrUndefined(
1397                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
1398     }
1399     
1400     void compileCompareStrictEq()
1401     {
1402         if (m_node->isBinaryUseKind(Int32Use)) {
1403             setBoolean(
1404                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1405             return;
1406         }
1407         
1408         if (m_node->isBinaryUseKind(NumberUse)) {
1409             setBoolean(
1410                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1411         }
1412         
1413         if (m_node->isBinaryUseKind(ObjectUse)) {
1414             masqueradesAsUndefinedWatchpointIfIsStillValid();
1415             setBoolean(
1416                 m_out.equal(
1417                     lowNonNullObject(m_node->child1()),
1418                     lowNonNullObject(m_node->child2())));
1419             return;
1420         }
1421         
1422         RELEASE_ASSERT_NOT_REACHED();
1423     }
1424     
1425     void compileCompareStrictEqConstant()
1426     {
1427         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
1428
1429         if (constant.isUndefinedOrNull()
1430             && !masqueradesAsUndefinedWatchpointIfIsStillValid()) {
1431             if (constant.isNull()) {
1432                 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull));
1433                 return;
1434             }
1435         
1436             ASSERT(constant.isUndefined());
1437             setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
1438             return;
1439         }
1440         
1441         setBoolean(
1442             m_out.equal(
1443                 lowJSValue(m_node->child1()),
1444                 m_out.constInt64(JSValue::encode(constant))));
1445     }
1446     
1447     void compileCompareLess()
1448     {
1449         if (m_node->isBinaryUseKind(Int32Use)) {
1450             setBoolean(
1451                 m_out.lessThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1452             return;
1453         }
1454         
1455         if (m_node->isBinaryUseKind(NumberUse)) {
1456             setBoolean(
1457                 m_out.doubleLessThan(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1458             return;
1459         }
1460         
1461         RELEASE_ASSERT_NOT_REACHED();
1462     }
1463     
1464     void compileCompareLessEq()
1465     {
1466         if (m_node->isBinaryUseKind(Int32Use)) {
1467             setBoolean(
1468                 m_out.lessThanOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1469             return;
1470         }
1471         
1472         if (m_node->isBinaryUseKind(NumberUse)) {
1473             setBoolean(
1474                 m_out.doubleLessThanOrEqual(
1475                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1476             return;
1477         }
1478         
1479         RELEASE_ASSERT_NOT_REACHED();
1480     }
1481     
1482     void compileCompareGreater()
1483     {
1484         if (m_node->isBinaryUseKind(Int32Use)) {
1485             setBoolean(
1486                 m_out.greaterThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1487             return;
1488         }
1489         
1490         if (m_node->isBinaryUseKind(NumberUse)) {
1491             setBoolean(
1492                 m_out.doubleGreaterThan(
1493                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1494             return;
1495         }
1496         
1497         RELEASE_ASSERT_NOT_REACHED();
1498     }
1499     
1500     void compileCompareGreaterEq()
1501     {
1502         if (m_node->isBinaryUseKind(Int32Use)) {
1503             setBoolean(
1504                 m_out.greaterThanOrEqual(
1505                     lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1506             return;
1507         }
1508         
1509         if (m_node->isBinaryUseKind(NumberUse)) {
1510             setBoolean(
1511                 m_out.doubleGreaterThanOrEqual(
1512                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1513             return;
1514         }
1515         
1516         RELEASE_ASSERT_NOT_REACHED();
1517     }
1518     
1519     void compileLogicalNot()
1520     {
1521         setBoolean(m_out.bitNot(boolify(m_node->child1())));
1522     }
1523     
1524     void compileJump()
1525     {
1526         m_out.jump(lowBlock(m_node->takenBlock()));
1527     }
1528     
1529     void compileBranch()
1530     {
1531         m_out.branch(
1532             boolify(m_node->child1()),
1533             lowBlock(m_node->takenBlock()),
1534             lowBlock(m_node->notTakenBlock()));
1535     }
1536     
1537     void compileSwitch()
1538     {
1539         SwitchData* data = m_node->switchData();
1540         switch (data->kind) {
1541         case SwitchImm: {
1542             Vector<ValueFromBlock, 2> intValues;
1543             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
1544             
1545             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
1546             
1547             switch (m_node->child1().useKind()) {
1548             case Int32Use: {
1549                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
1550                 m_out.jump(switchOnInts);
1551                 break;
1552             }
1553                 
1554             case UntypedUse: {
1555                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
1556                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
1557                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
1558                 
1559                 LValue boxedValue = lowJSValue(m_node->child1());
1560                 m_out.branch(isNotInt32(boxedValue), isNotInt, isInt);
1561                 
1562                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
1563                 
1564                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
1565                 m_out.jump(switchOnInts);
1566                 
1567                 m_out.appendTo(isNotInt, isDouble);
1568                 m_out.branch(
1569                     isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble);
1570                 
1571                 m_out.appendTo(isDouble, innerLastNext);
1572                 LValue doubleValue = unboxDouble(boxedValue);
1573                 LValue intInDouble = m_out.fpToInt32(doubleValue);
1574                 intValues.append(m_out.anchor(intInDouble));
1575                 m_out.branch(
1576                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
1577                     switchOnInts, lowBlock(data->fallThrough));
1578                 break;
1579             }
1580                 
1581             default:
1582                 RELEASE_ASSERT_NOT_REACHED();
1583                 break;
1584             }
1585             
1586             m_out.appendTo(switchOnInts, lastNext);
1587             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
1588             return;
1589         }
1590         
1591         case SwitchChar: {
1592             LValue stringValue;
1593             
1594             switch (m_node->child1().useKind()) {
1595             case StringUse: {
1596                 stringValue = lowString(m_node->child1());
1597                 break;
1598             }
1599                 
1600             case UntypedUse: {
1601                 LValue unboxedValue = lowJSValue(m_node->child1());
1602                 
1603                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
1604                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
1605                 
1606                 m_out.branch(
1607                     isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase);
1608                 
1609                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
1610                 LValue cellValue = unboxedValue;
1611                 m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase);
1612                 
1613                 m_out.appendTo(isStringCase, lastNext);
1614                 stringValue = cellValue;
1615                 break;
1616             }
1617                 
1618             default:
1619                 RELEASE_ASSERT_NOT_REACHED();
1620                 break;
1621             }
1622             
1623             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
1624             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
1625             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
1626             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
1627             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
1628             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
1629             
1630             m_out.branch(
1631                 m_out.notEqual(
1632                     m_out.load32(stringValue, m_heaps.JSString_length),
1633                     m_out.int32One),
1634                 lowBlock(data->fallThrough), lengthIs1);
1635             
1636             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
1637             Vector<ValueFromBlock, 2> values;
1638             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
1639             values.append(m_out.anchor(fastValue));
1640             m_out.branch(m_out.isNull(fastValue), needResolution, resolved);
1641             
1642             m_out.appendTo(needResolution, resolved);
1643             values.append(m_out.anchor(
1644                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
1645             m_out.jump(resolved);
1646             
1647             m_out.appendTo(resolved, is8Bit);
1648             LValue value = m_out.phi(m_out.intPtr, values);
1649             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
1650             m_out.branch(
1651                 m_out.testNonZero32(
1652                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
1653                     m_out.constInt32(StringImpl::flagIs8Bit())),
1654                 is8Bit, is16Bit);
1655             
1656             Vector<ValueFromBlock, 2> characters;
1657             m_out.appendTo(is8Bit, is16Bit);
1658             characters.append(m_out.anchor(
1659                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
1660             m_out.jump(continuation);
1661             
1662             m_out.appendTo(is16Bit, continuation);
1663             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
1664             m_out.jump(continuation);
1665             
1666             m_out.appendTo(continuation, lastNext);
1667             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
1668             return;
1669         }
1670         
1671         case SwitchString:
1672             RELEASE_ASSERT_NOT_REACHED();
1673             break;
1674         }
1675         
1676         RELEASE_ASSERT_NOT_REACHED();
1677     }
1678     
1679     void compileReturn()
1680     {
1681         // FIXME: have a real epilogue when we switch to using our calling convention.
1682         // https://bugs.webkit.org/show_bug.cgi?id=113621
1683         m_out.ret(lowJSValue(m_node->child1()));
1684     }
1685     
1686     void compileForceOSRExit()
1687     {
1688         terminate(InadequateCoverage);
1689     }
1690     
1691     LValue boolify(Edge edge)
1692     {
1693         switch (edge.useKind()) {
1694         case BooleanUse:
1695             return lowBoolean(m_node->child1());
1696         case Int32Use:
1697             return m_out.notZero32(lowInt32(m_node->child1()));
1698         case NumberUse:
1699             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
1700         case ObjectOrOtherUse:
1701             return m_out.bitNot(
1702                 equalNullOrUndefined(
1703                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
1704                     ManualOperandSpeculation));
1705         default:
1706             RELEASE_ASSERT_NOT_REACHED();
1707             return 0;
1708         }
1709     }
1710     
1711     enum StringOrObjectMode {
1712         AllCellsAreFalse,
1713         CellCaseSpeculatesObject
1714     };
1715     enum EqualNullOrUndefinedMode {
1716         EqualNull,
1717         EqualUndefined,
1718         EqualNullOrUndefined,
1719         SpeculateNullOrUndefined
1720     };
1721     LValue equalNullOrUndefined(
1722         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
1723         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
1724     {
1725         bool validWatchpoint = masqueradesAsUndefinedWatchpointIfIsStillValid();
1726         
1727         LValue value = lowJSValue(edge, operandMode);
1728         
1729         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
1730         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
1731         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
1732         
1733         m_out.branch(isNotCell(value), primitiveCase, cellCase);
1734         
1735         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
1736         
1737         Vector<ValueFromBlock, 3> results;
1738         
1739         switch (cellMode) {
1740         case AllCellsAreFalse:
1741             break;
1742         case CellCaseSpeculatesObject:
1743             FTL_TYPE_CHECK(
1744                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
1745                 m_out.equal(
1746                     m_out.loadPtr(value, m_heaps.JSCell_structure),
1747                     m_out.constIntPtr(vm().stringStructure.get())));
1748             break;
1749         }
1750         
1751         if (validWatchpoint) {
1752             results.append(m_out.anchor(m_out.booleanFalse));
1753             m_out.jump(continuation);
1754         } else {
1755             LBasicBlock masqueradesCase =
1756                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
1757                 
1758             LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure);
1759             
1760             results.append(m_out.anchor(m_out.booleanFalse));
1761             
1762             m_out.branch(
1763                 m_out.testNonZero8(
1764                     m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
1765                     m_out.constInt8(MasqueradesAsUndefined)),
1766                 masqueradesCase, continuation);
1767             
1768             m_out.appendTo(masqueradesCase, primitiveCase);
1769             
1770             results.append(m_out.anchor(
1771                 m_out.equal(
1772                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)),
1773                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
1774             m_out.jump(continuation);
1775         }
1776         
1777         m_out.appendTo(primitiveCase, continuation);
1778         
1779         LValue primitiveResult;
1780         switch (primitiveMode) {
1781         case EqualNull:
1782             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
1783             break;
1784         case EqualUndefined:
1785             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
1786             break;
1787         case EqualNullOrUndefined:
1788             primitiveResult = m_out.equal(
1789                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1790                 m_out.constInt64(ValueNull));
1791             break;
1792         case SpeculateNullOrUndefined:
1793             FTL_TYPE_CHECK(
1794                 jsValueValue(value), edge, SpecCell | SpecOther,
1795                 m_out.notEqual(
1796                     m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1797                     m_out.constInt64(ValueNull)));
1798             primitiveResult = m_out.booleanTrue;
1799             break;
1800         }
1801         results.append(m_out.anchor(primitiveResult));
1802         m_out.jump(continuation);
1803         
1804         m_out.appendTo(continuation, lastNext);
1805         
1806         return m_out.phi(m_out.boolean, results);
1807     }
1808     
1809     template<typename FunctionType>
1810     void contiguousPutByValOutOfBounds(
1811         FunctionType slowPathFunction,
1812         LValue base, LValue storage, LValue index, LValue value,
1813         LBasicBlock continuation)
1814     {
1815         LValue isNotInBounds = m_out.aboveOrEqual(
1816             index, m_out.load32(storage, m_heaps.Butterfly_publicLength));
1817         if (m_node->arrayMode().isInBounds())
1818             speculate(StoreToHoleOrOutOfBounds, noValue(), 0, isNotInBounds);
1819         else {
1820             LBasicBlock notInBoundsCase =
1821                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
1822             LBasicBlock performStore =
1823                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
1824                 
1825             m_out.branch(isNotInBounds, notInBoundsCase, performStore);
1826                 
1827             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
1828                 
1829             LValue isOutOfBounds = m_out.aboveOrEqual(
1830                 index, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
1831                 
1832             if (!m_node->arrayMode().isOutOfBounds())
1833                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
1834             else {
1835                 LBasicBlock outOfBoundsCase =
1836                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
1837                 LBasicBlock holeCase =
1838                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
1839                     
1840                 m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase);
1841                     
1842                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
1843                     
1844                 vmCall(
1845                     m_out.operation(slowPathFunction),
1846                     m_callFrame, base, index, value);
1847                     
1848                 m_out.jump(continuation);
1849                     
1850                 m_out.appendTo(holeCase, innerLastNext);
1851             }
1852             
1853             m_out.store32(
1854                 m_out.add(index, m_out.int32One),
1855                 storage, m_heaps.Butterfly_publicLength);
1856                 
1857             m_out.jump(performStore);
1858             m_out.appendTo(performStore, lastNext);
1859         }
1860     }
1861     
1862     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
1863     {
1864         Vector<SwitchCase> cases;
1865         for (unsigned i = 0; i < data->cases.size(); ++i) {
1866             cases.append(SwitchCase(
1867                 constInt(type, data->cases[i].value.switchLookupValue()),
1868                 lowBlock(data->cases[i].target)));
1869         }
1870         
1871         m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough));
1872     }
1873     
1874     void speculateBackward(
1875         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1876     {
1877         appendOSRExit(
1878             kind, lowValue, highValue, failCondition, BackwardSpeculation, FormattedValue());
1879     }
1880     
1881     void speculateForward(
1882         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
1883         const FormattedValue& recovery)
1884     {
1885         appendOSRExit(
1886             kind, lowValue, highValue, failCondition, ForwardSpeculation, recovery);
1887     }
1888     
1889     void speculate(
1890         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1891     {
1892         appendOSRExit(
1893             kind, lowValue, highValue, failCondition, m_direction, FormattedValue());
1894     }
1895     
1896     void terminate(ExitKind kind)
1897     {
1898         speculate(kind, noValue(), 0, m_out.booleanTrue);
1899     }
1900     
1901     void backwardTypeCheck(
1902         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1903         LValue failCondition)
1904     {
1905         appendTypeCheck(
1906             lowValue, highValue, typesPassedThrough, failCondition, BackwardSpeculation,
1907             FormattedValue());
1908     }
1909     
1910     void forwardTypeCheck(
1911         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1912         LValue failCondition)
1913     {
1914         appendTypeCheck(
1915             lowValue, highValue, typesPassedThrough, failCondition, ForwardSpeculation,
1916             FormattedValue());
1917     }
1918     
1919     void typeCheck(
1920         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1921         LValue failCondition)
1922     {
1923         appendTypeCheck(
1924             lowValue, highValue, typesPassedThrough, failCondition, m_direction,
1925             FormattedValue());
1926     }
1927     
1928     void appendTypeCheck(
1929         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1930         LValue failCondition, SpeculationDirection direction, FormattedValue recovery)
1931     {
1932         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
1933             return;
1934         ASSERT(mayHaveTypeCheck(highValue.useKind()));
1935         appendOSRExit(BadType, lowValue, highValue.node(), failCondition, direction, recovery);
1936         m_interpreter.filter(highValue, typesPassedThrough);
1937     }
1938     
1939     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1940     {
1941         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
1942         
1943         LoweredNodeValue value = m_int32Values.get(edge.node());
1944         if (isValid(value))
1945             return value.value();
1946         
1947         value = m_jsValueValues.get(edge.node());
1948         if (isValid(value)) {
1949             LValue boxedResult = value.value();
1950             FTL_TYPE_CHECK(
1951                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
1952             LValue result = unboxInt32(boxedResult);
1953             setInt32(edge.node(), result);
1954             return result;
1955         }
1956
1957         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
1958         terminate(Uncountable);
1959         return m_out.int32Zero;
1960     }
1961     
1962     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1963     {
1964         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
1965         
1966         LoweredNodeValue value = m_jsValueValues.get(edge.node());
1967         if (isValid(value)) {
1968             LValue uncheckedValue = value.value();
1969             FTL_TYPE_CHECK(
1970                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
1971             return uncheckedValue;
1972         }
1973         
1974         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
1975         terminate(Uncountable);
1976         return m_out.intPtrZero;
1977     }
1978     
1979     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1980     {
1981         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
1982         
1983         LValue result = lowCell(edge, mode);
1984         speculateObject(edge, result);
1985         return result;
1986     }
1987     
1988     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1989     {
1990         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse);
1991         
1992         LValue result = lowCell(edge, mode);
1993         speculateString(edge, result);
1994         return result;
1995     }
1996     
1997     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1998     {
1999         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2000         
2001         LValue result = lowCell(edge, mode);
2002         speculateNonNullObject(edge, result);
2003         return result;
2004     }
2005     
2006     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2007     {
2008         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
2009         
2010         LoweredNodeValue value = m_booleanValues.get(edge.node());
2011         if (isValid(value))
2012             return value.value();
2013         
2014         value = m_jsValueValues.get(edge.node());
2015         if (isValid(value)) {
2016             LValue unboxedResult = value.value();
2017             FTL_TYPE_CHECK(
2018                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
2019             LValue result = unboxBoolean(unboxedResult);
2020             setBoolean(edge.node(), result);
2021             return result;
2022         }
2023         
2024         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
2025         terminate(Uncountable);
2026         return m_out.booleanFalse;
2027     }
2028     
2029     LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2030     {
2031         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind()));
2032         
2033         LoweredNodeValue value = m_doubleValues.get(edge.node());
2034         if (isValid(value))
2035             return value.value();
2036         
2037         value = m_int32Values.get(edge.node());
2038         if (isValid(value)) {
2039             LValue result = m_out.intToDouble(value.value());
2040             setDouble(edge.node(), result);
2041             return result;
2042         }
2043         
2044         value = m_jsValueValues.get(edge.node());
2045         if (isValid(value)) {
2046             LValue boxedResult = value.value();
2047             
2048             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case"));
2049             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case"));
2050             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation"));
2051             
2052             m_out.branch(isNotInt32(boxedResult), doubleCase, intCase);
2053             
2054             LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
2055             
2056             ValueFromBlock intToDouble = m_out.anchor(
2057                 m_out.intToDouble(unboxInt32(boxedResult)));
2058             m_out.jump(continuation);
2059             
2060             m_out.appendTo(doubleCase, continuation);
2061             
2062             FTL_TYPE_CHECK(
2063                 jsValueValue(boxedResult), edge, SpecNumber, isCellOrMisc(boxedResult));
2064             
2065             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult));
2066             m_out.jump(continuation);
2067             
2068             m_out.appendTo(continuation, lastNext);
2069             
2070             LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
2071             
2072             setDouble(edge.node(), result);
2073             return result;
2074         }
2075         
2076         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecNumber));
2077         terminate(Uncountable);
2078         return m_out.doubleZero;
2079     }
2080     
2081     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2082     {
2083         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2084         
2085         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2086         if (isValid(value))
2087             return value.value();
2088         
2089         value = m_int32Values.get(edge.node());
2090         if (isValid(value)) {
2091             LValue result = boxInt32(value.value());
2092             setJSValue(edge.node(), result);
2093             return result;
2094         }
2095         
2096         value = m_booleanValues.get(edge.node());
2097         if (isValid(value)) {
2098             LValue result = boxBoolean(value.value());
2099             setJSValue(edge.node(), result);
2100             return result;
2101         }
2102         
2103         value = m_doubleValues.get(edge.node());
2104         if (isValid(value)) {
2105             LValue result = boxDouble(value.value());
2106             setJSValue(edge.node(), result);
2107             return result;
2108         }
2109         
2110         RELEASE_ASSERT_NOT_REACHED();
2111         return 0;
2112     }
2113     
2114     LValue lowStorage(Edge edge)
2115     {
2116         LoweredNodeValue value = m_storageValues.get(edge.node());
2117         if (isValid(value))
2118             return value.value();
2119         
2120         LValue result = lowCell(edge);
2121         setStorage(edge.node(), result);
2122         return result;
2123     }
2124     
2125     LValue isNotInt32(LValue jsValue)
2126     {
2127         return m_out.below(jsValue, m_tagTypeNumber);
2128     }
2129     LValue unboxInt32(LValue jsValue)
2130     {
2131         return m_out.castToInt32(jsValue);
2132     }
2133     LValue boxInt32(LValue value)
2134     {
2135         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
2136     }
2137     
2138     LValue isCellOrMisc(LValue jsValue)
2139     {
2140         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
2141     }
2142     LValue unboxDouble(LValue jsValue)
2143     {
2144         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
2145     }
2146     LValue boxDouble(LValue doubleValue)
2147     {
2148         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
2149     }
2150     
2151     LValue isNotCell(LValue jsValue)
2152     {
2153         return m_out.testNonZero64(jsValue, m_tagMask);
2154     }
2155     
2156     LValue isNotBoolean(LValue jsValue)
2157     {
2158         return m_out.testNonZero64(
2159             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
2160             m_out.constInt64(~1));
2161     }
2162     LValue unboxBoolean(LValue jsValue)
2163     {
2164         // We want to use a cast that guarantees that LLVM knows that even the integer
2165         // value is just 0 or 1. But for now we do it the dumb way.
2166         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
2167     }
2168     LValue boxBoolean(LValue value)
2169     {
2170         return m_out.select(
2171             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
2172     }
2173
2174     void speculate(Edge edge)
2175     {
2176         switch (edge.useKind()) {
2177         case UntypedUse:
2178             break;
2179         case KnownInt32Use:
2180         case KnownNumberUse:
2181             ASSERT(!m_interpreter.needsTypeCheck(edge));
2182             break;
2183         case Int32Use:
2184             speculateInt32(edge);
2185             break;
2186         case CellUse:
2187             speculateCell(edge);
2188             break;
2189         case KnownCellUse:
2190             ASSERT(!m_interpreter.needsTypeCheck(edge));
2191             break;
2192         case ObjectUse:
2193             speculateObject(edge);
2194             break;
2195         case StringUse:
2196             speculateString(edge);
2197             break;
2198         case RealNumberUse:
2199             speculateRealNumber(edge);
2200             break;
2201         case NumberUse:
2202             speculateNumber(edge);
2203             break;
2204         case BooleanUse:
2205             speculateBoolean(edge);
2206             break;
2207         default:
2208             RELEASE_ASSERT_NOT_REACHED();
2209         }
2210     }
2211     
2212     void speculate(Node*, Edge edge)
2213     {
2214         speculate(edge);
2215     }
2216     
2217     void speculateInt32(Edge edge)
2218     {
2219         lowInt32(edge);
2220     }
2221     
2222     void speculateCell(Edge edge)
2223     {
2224         lowCell(edge);
2225     }
2226     
2227     LValue isObject(LValue cell)
2228     {
2229         return m_out.notEqual(
2230             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2231             m_out.constIntPtr(vm().stringStructure.get()));
2232     }
2233     
2234     LValue isNotString(LValue cell)
2235     {
2236         return isObject(cell);
2237     }
2238     
2239     LValue isString(LValue cell)
2240     {
2241         return m_out.equal(
2242             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2243             m_out.constIntPtr(vm().stringStructure.get()));
2244     }
2245     
2246     LValue isNotObject(LValue cell)
2247     {
2248         return isString(cell);
2249     }
2250     
2251     void speculateObject(Edge edge, LValue cell)
2252     {
2253         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
2254     }
2255     
2256     void speculateObject(Edge edge)
2257     {
2258         speculateObject(edge, lowCell(edge));
2259     }
2260     
2261     void speculateString(Edge edge, LValue cell)
2262     {
2263         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
2264     }
2265     
2266     void speculateString(Edge edge)
2267     {
2268         speculateString(edge, lowCell(edge));
2269     }
2270     
2271     void speculateNonNullObject(Edge edge, LValue cell)
2272     {
2273         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
2274         FTL_TYPE_CHECK(
2275             jsValueValue(cell), edge, SpecObject, 
2276             m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get())));
2277         if (masqueradesAsUndefinedWatchpointIfIsStillValid())
2278             return;
2279         
2280         speculate(
2281             BadType, jsValueValue(cell), edge.node(),
2282             m_out.testNonZero8(
2283                 m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2284                 m_out.constInt8(MasqueradesAsUndefined)));
2285     }
2286     
2287     void speculateNumber(Edge edge)
2288     {
2289         // Do an early return here because lowDouble() can create a lot of control flow.
2290         if (!m_interpreter.needsTypeCheck(edge))
2291             return;
2292         
2293         lowDouble(edge);
2294     }
2295     
2296     void speculateRealNumber(Edge edge)
2297     {
2298         // Do an early return here because lowDouble() can create a lot of control flow.
2299         if (!m_interpreter.needsTypeCheck(edge))
2300             return;
2301         
2302         LValue value = lowDouble(edge);
2303         FTL_TYPE_CHECK(
2304             doubleValue(value), edge, SpecRealNumber,
2305             m_out.doubleNotEqualOrUnordered(value, value));
2306     }
2307     
2308     void speculateBoolean(Edge edge)
2309     {
2310         lowBoolean(edge);
2311     }
2312
2313     bool masqueradesAsUndefinedWatchpointIsStillValid()
2314     {
2315         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin);
2316     }
2317     
2318     bool masqueradesAsUndefinedWatchpointIfIsStillValid()
2319     {
2320         if (!masqueradesAsUndefinedWatchpointIsStillValid())
2321             return false;
2322         
2323         // FIXME: Implement masquerades-as-undefined watchpoints.
2324         // https://bugs.webkit.org/show_bug.cgi?id=113647
2325         return true;
2326     }
2327     
2328     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
2329     
2330     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
2331     {
2332         callPreflight();
2333         LValue result = m_out.call(function);
2334         callCheck(mode);
2335         return result;
2336     }
2337     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
2338     {
2339         callPreflight();
2340         LValue result = m_out.call(function, arg1);
2341         callCheck(mode);
2342         return result;
2343     }
2344     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
2345     {
2346         callPreflight();
2347         LValue result = m_out.call(function, arg1, arg2);
2348         callCheck(mode);
2349         return result;
2350     }
2351     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
2352     {
2353         callPreflight();
2354         LValue result = m_out.call(function, arg1, arg2, arg3);
2355         callCheck(mode);
2356         return result;
2357     }
2358     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
2359     {
2360         callPreflight();
2361         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
2362         callCheck(mode);
2363         return result;
2364     }
2365     
2366     void callPreflight(CodeOrigin codeOrigin)
2367     {
2368         m_out.store32(
2369             m_out.constInt32(
2370                 CallFrame::Location::encodeAsCodeOriginIndex(
2371                     codeBlock()->addCodeOrigin(codeOrigin))), 
2372             tagFor(JSStack::ArgumentCount));
2373     }
2374     void callPreflight()
2375     {
2376         callPreflight(m_node->codeOrigin);
2377     }
2378     
2379     void callCheck(ExceptionCheckMode mode = CheckExceptions)
2380     {
2381         if (mode == NoExceptions)
2382             return;
2383         
2384         LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception"));
2385         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
2386         
2387         m_out.branch(
2388             m_out.notZero64(m_out.load64(m_out.absolute(&vm().exception))),
2389             didHaveException, continuation);
2390         
2391         LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation);
2392         // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622
2393         m_out.crash();
2394         
2395         m_out.appendTo(continuation, lastNext);
2396     }
2397     
2398     bool isLive(Node* node)
2399     {
2400         return m_live.contains(node);
2401     }
2402     
2403     void use(Edge edge)
2404     {
2405         ASSERT(edge->hasResult());
2406         if (!edge.doesKill())
2407             return;
2408         m_live.remove(edge.node());
2409     }
2410     
2411     // Wrapper used only for DFG_NODE_DO_TO_CHILDREN
2412     void use(Node*, Edge edge)
2413     {
2414         use(edge);
2415     }
2416     
2417     LBasicBlock lowBlock(BasicBlock* block)
2418     {
2419         return m_blocks.get(block);
2420     }
2421     
2422     void initializeOSRExitStateForBlock()
2423     {
2424         for (unsigned i = m_valueSources.size(); i--;) {
2425             FlushFormat format = m_highBlock->ssa->flushFormatAtHead[i];
2426             switch (format) {
2427             case DeadFlush: {
2428                 // Must consider available nodes instead.
2429                 Node* node = m_highBlock->ssa->availabilityAtHead[i];
2430                 if (!node) {
2431                     m_valueSources[i] = ValueSource(SourceIsDead);
2432                     break;
2433                 }
2434                 
2435                 m_valueSources[i] = ValueSource(node);
2436                 break;
2437             }
2438                 
2439             case FlushedInt32:
2440                 m_valueSources[i] = ValueSource(Int32InJSStack);
2441                 break;
2442                 
2443             case FlushedDouble:
2444                 m_valueSources[i] = ValueSource(DoubleInJSStack);
2445                 break;
2446                 
2447             case FlushedCell:
2448             case FlushedBoolean:
2449             case FlushedJSValue:
2450                 m_valueSources[i] = ValueSource(ValueInJSStack);
2451                 break;
2452             }
2453         }
2454     }
2455     
2456     void appendOSRExit(
2457         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2458         SpeculationDirection direction, FormattedValue recovery)
2459     {
2460         if (Options::ftlTrapsOnOSRExit()) {
2461             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase"));
2462             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation"));
2463             
2464             m_out.branch(failCondition, failCase, continuation);
2465             
2466             LBasicBlock lastNext = m_out.appendTo(failCase, continuation);
2467             m_out.trap();
2468             m_out.unreachable();
2469             
2470             m_out.appendTo(continuation, lastNext);
2471             return;
2472         }
2473         
2474         if (verboseCompilationEnabled())
2475             dataLog("    OSR exit with value sources: ", m_valueSources, "\n");
2476         
2477         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.osrExit.size());
2478         unsigned index = m_ftlState.osrExit.size();
2479         
2480         m_ftlState.jitCode->osrExit.append(OSRExit(
2481             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
2482             m_codeOriginForExitTarget, m_codeOriginForExitProfile, m_lastSetOperand,
2483             m_valueSources.numberOfArguments(), m_valueSources.numberOfLocals()));
2484         m_ftlState.osrExit.append(OSRExitCompilationInfo());
2485         
2486         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
2487         OSRExitCompilationInfo& info = m_ftlState.osrExit.last();
2488
2489         LBasicBlock lastNext = 0;
2490         LBasicBlock continuation = 0;
2491         
2492         if (!Options::useLLVMOSRExitIntrinsic()) {
2493             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase"));
2494             continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation"));
2495             
2496             m_out.branch(failCondition, failCase, continuation);
2497
2498             m_out.appendTo(m_prologue);
2499             info.m_thunkAddress = buildAlloca(m_out.m_builder, m_out.intPtr);
2500         
2501             lastNext = m_out.appendTo(failCase, continuation);
2502         }
2503         
2504         if (Options::ftlOSRExitOmitsMarshalling()) {
2505             m_out.call(
2506                 m_out.intToPtr(
2507                     m_out.get(info.m_thunkAddress),
2508                     pointerType(functionType(m_out.voidType))));
2509         } else
2510             emitOSRExitCall(failCondition, index, exit, info, lowValue, direction, recovery);
2511         
2512         if (!Options::useLLVMOSRExitIntrinsic()) {
2513             m_out.unreachable();
2514             
2515             m_out.appendTo(continuation, lastNext);
2516         
2517             m_exitThunkGenerator.emitThunk(index);
2518         }
2519     }
2520     
2521     void emitOSRExitCall(
2522         LValue failCondition, unsigned index, OSRExit& exit, OSRExitCompilationInfo& info,
2523         FormattedValue lowValue, SpeculationDirection direction, FormattedValue recovery)
2524     {
2525         ExitArgumentList arguments;
2526         
2527         if (Options::useLLVMOSRExitIntrinsic()) {
2528             arguments.append(failCondition);
2529             arguments.append(m_out.constInt32(index));
2530         }
2531         
2532         arguments.append(m_callFrame);
2533         if (!!lowValue)
2534             arguments.append(lowValue.value());
2535         
2536         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
2537             ValueSource source = m_valueSources[i];
2538             
2539             switch (source.kind()) {
2540             case ValueInJSStack:
2541                 exit.m_values[i] = ExitValue::inJSStack();
2542                 break;
2543             case Int32InJSStack:
2544                 exit.m_values[i] = ExitValue::inJSStackAsInt32();
2545                 break;
2546             case DoubleInJSStack:
2547                 exit.m_values[i] = ExitValue::inJSStackAsDouble();
2548                 break;
2549             case SourceIsDead:
2550                 exit.m_values[i] = ExitValue::dead();
2551                 break;
2552             case HaveNode:
2553                 addExitArgumentForNode(exit, arguments, i, source.node());
2554                 break;
2555             default:
2556                 RELEASE_ASSERT_NOT_REACHED();
2557                 break;
2558             }
2559         }
2560         
2561         if (verboseCompilationEnabled())
2562             dataLog("        Exit values: ", exit.m_values, "\n");
2563         
2564         if (direction == ForwardSpeculation) {
2565             ASSERT(m_node);
2566             exit.convertToForward(m_highBlock, m_node, m_nodeIndex, recovery, arguments);
2567         }
2568         
2569         // So, the really lame thing here is that we have to build an LLVM function type.
2570         // Booo.
2571         Vector<LType, 16> argumentTypes;
2572         for (unsigned i = 0; i < arguments.size(); ++i)
2573             argumentTypes.append(typeOf(arguments[i]));
2574         
2575         if (Options::useLLVMOSRExitIntrinsic()) {
2576             m_out.call(m_out.osrExitIntrinsic(), arguments);
2577             return;
2578         }
2579         
2580         m_out.call(
2581             m_out.intToPtr(
2582                 m_out.get(info.m_thunkAddress),
2583                 pointerType(functionType(m_out.voidType, argumentTypes))),
2584             arguments);
2585     }
2586     
2587     void addExitArgumentForNode(
2588         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
2589     {
2590         ASSERT(node->shouldGenerate());
2591         ASSERT(node->hasResult());
2592
2593         if (tryToSetConstantExitArgument(exit, index, node))
2594             return;
2595         
2596         if (!isLive(node)) {
2597             bool found = false;
2598             
2599             if (needsOSRBackwardRewiring(node->op())) {
2600                 node = node->child1().node();
2601                 if (tryToSetConstantExitArgument(exit, index, node))
2602                     return;
2603                 if (isLive(node))
2604                     found = true;
2605             }
2606             
2607             if (!found) {
2608                 Node* int32ToDouble = 0;
2609                 Node* valueToInt32 = 0;
2610                 Node* uint32ToNumber = 0;
2611                 Node* doubleAsInt32 = 0;
2612                 
2613                 HashSet<Node*>::iterator iter = m_live.begin();
2614                 HashSet<Node*>::iterator end = m_live.end();
2615                 for (; iter != end; ++iter) {
2616                     Node* candidate = *iter;
2617                     if (!candidate->child1())
2618                         continue;
2619                     if (candidate->child1() != node)
2620                         continue;
2621                     switch (candidate->op()) {
2622                     case Int32ToDouble:
2623                         int32ToDouble = candidate;
2624                         break;
2625                     case ValueToInt32:
2626                         valueToInt32 = candidate;
2627                         break;
2628                     case UInt32ToNumber:
2629                         uint32ToNumber = candidate;
2630                         break;
2631                     case DoubleAsInt32:
2632                         uint32ToNumber = candidate;
2633                         break;
2634                     default:
2635                         ASSERT(!needsOSRForwardRewiring(candidate->op()));
2636                         break;
2637                     }
2638                 }
2639                 
2640                 if (doubleAsInt32)
2641                     node = doubleAsInt32;
2642                 else if (int32ToDouble)
2643                     node = int32ToDouble;
2644                 else if (valueToInt32)
2645                     node = valueToInt32;
2646                 else if (uint32ToNumber)
2647                     node = uint32ToNumber;
2648                 
2649                 if (isLive(node))
2650                     found = true;
2651             }
2652             
2653             if (!found) {
2654                 exit.m_values[index] = ExitValue::dead();
2655                 return;
2656             }
2657         }
2658         
2659         ASSERT(isLive(node));
2660
2661         LoweredNodeValue value = m_int32Values.get(node);
2662         if (isValid(value)) {
2663             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
2664             return;
2665         }
2666         
2667         value = m_booleanValues.get(node);
2668         if (isValid(value)) {
2669             addExitArgument(exit, arguments, index, ValueFormatBoolean, value.value());
2670             return;
2671         }
2672         
2673         value = m_jsValueValues.get(node);
2674         if (isValid(value)) {
2675             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
2676             return;
2677         }
2678         
2679         value = m_doubleValues.get(node);
2680         if (isValid(value)) {
2681             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
2682             return;
2683         }
2684
2685         dataLog("Cannot find value for node: ", node, "\n");
2686         RELEASE_ASSERT_NOT_REACHED();
2687     }
2688     
2689     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
2690     {
2691         if (!node)
2692             return false;
2693         
2694         switch (node->op()) {
2695         case JSConstant:
2696         case WeakJSConstant:
2697             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
2698             return true;
2699         case PhantomArguments:
2700             // FIXME: implement PhantomArguments.
2701             // https://bugs.webkit.org/show_bug.cgi?id=113986
2702             RELEASE_ASSERT_NOT_REACHED();
2703             return true;
2704         default:
2705             return false;
2706         }
2707     }
2708     
2709     void addExitArgument(
2710         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
2711         LValue value)
2712     {
2713         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
2714         arguments.append(value);
2715     }
2716     
2717     void linkOSRExitsAndCompleteInitializationBlocks()
2718     {
2719         MacroAssemblerCodeRef osrExitThunk =
2720             vm().getCTIStub(osrExitGenerationThunkGenerator);
2721         CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
2722         
2723         m_out.appendTo(m_prologue);
2724         m_out.jump(m_initialization);
2725         
2726         m_out.appendTo(m_initialization);
2727         
2728         if (m_exitThunkGenerator.didThings()) {
2729             OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
2730                 vm(), &m_exitThunkGenerator, m_ftlState.graph.m_codeBlock,
2731                 JITCompilationMustSucceed));
2732         
2733             ASSERT(m_ftlState.osrExit.size() == m_ftlState.jitCode->osrExit.size());
2734         
2735             for (unsigned i = 0; i < m_ftlState.osrExit.size(); ++i) {
2736                 OSRExitCompilationInfo& info = m_ftlState.osrExit[i];
2737                 OSRExit& exit = m_ftlState.jitCode->osrExit[i];
2738             
2739                 linkBuffer->link(info.m_thunkJump, target);
2740             
2741                 m_out.set(
2742                     m_out.constIntPtr(
2743                         linkBuffer->locationOf(info.m_thunkLabel).executableAddress()),
2744                     info.m_thunkAddress);
2745             
2746                 exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
2747             }
2748         
2749             m_ftlState.finalizer->initializeExitThunksLinkBuffer(linkBuffer.release());
2750         }
2751
2752         m_out.jump(lowBlock(m_graph.block(0)));
2753     }
2754     
2755     void observeMovHint(Node* node)
2756     {
2757         ASSERT(node->containsMovHint());
2758         ASSERT(node->op() != ZombieHint);
2759         
2760         int operand = node->local();
2761         
2762         m_lastSetOperand = operand;
2763         m_valueSources.operand(operand) = ValueSource(node->child1().node());
2764     }
2765     
2766     void setInt32(Node* node, LValue value)
2767     {
2768         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
2769     }
2770     void setJSValue(Node* node, LValue value)
2771     {
2772         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
2773     }
2774     void setBoolean(Node* node, LValue value)
2775     {
2776         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
2777     }
2778     void setStorage(Node* node, LValue value)
2779     {
2780         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
2781     }
2782     void setDouble(Node* node, LValue value)
2783     {
2784         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
2785     }
2786
2787     void setInt32(LValue value)
2788     {
2789         setInt32(m_node, value);
2790     }
2791     void setJSValue(LValue value)
2792     {
2793         setJSValue(m_node, value);
2794     }
2795     void setBoolean(LValue value)
2796     {
2797         setBoolean(m_node, value);
2798     }
2799     void setStorage(LValue value)
2800     {
2801         setStorage(m_node, value);
2802     }
2803     void setDouble(LValue value)
2804     {
2805         setDouble(m_node, value);
2806     }
2807     
2808     bool isValid(const LoweredNodeValue& value)
2809     {
2810         if (!value)
2811             return false;
2812         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
2813             return false;
2814         return true;
2815     }
2816     
2817     void addWeakReference(JSCell* target)
2818     {
2819         m_ftlState.jitCode->common.weakReferences.append(
2820             WriteBarrier<JSCell>(vm(), codeBlock()->ownerExecutable(), target));
2821     }
2822     
2823     LValue weakPointer(JSCell* pointer)
2824     {
2825         addWeakReference(pointer);
2826         return m_out.constIntPtr(pointer);
2827     }
2828     
2829     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
2830     {
2831         return m_out.address(base, m_heaps.variables[operand], offset);
2832     }
2833     TypedPointer payloadFor(LValue base, int operand)
2834     {
2835         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
2836     }
2837     TypedPointer tagFor(LValue base, int operand)
2838     {
2839         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
2840     }
2841     TypedPointer addressFor(int operand)
2842     {
2843         return addressFor(m_callFrame, operand);
2844     }
2845     TypedPointer payloadFor(int operand)
2846     {
2847         return payloadFor(m_callFrame, operand);
2848     }
2849     TypedPointer tagFor(int operand)
2850     {
2851         return tagFor(m_callFrame, operand);
2852     }
2853     
2854     VM& vm() { return m_graph.m_vm; }
2855     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
2856     
2857     Graph& m_graph;
2858     State& m_ftlState;
2859     AbstractHeapRepository m_heaps;
2860     Output m_out;
2861     
2862     LBasicBlock m_prologue;
2863     LBasicBlock m_initialization;
2864     HashMap<BasicBlock*, LBasicBlock> m_blocks;
2865     
2866     LValue m_callFrame;
2867     LValue m_tagTypeNumber;
2868     LValue m_tagMask;
2869     
2870     HashMap<Node*, LoweredNodeValue> m_int32Values;
2871     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
2872     HashMap<Node*, LoweredNodeValue> m_booleanValues;
2873     HashMap<Node*, LoweredNodeValue> m_storageValues;
2874     HashMap<Node*, LoweredNodeValue> m_doubleValues;
2875     HashSet<Node*> m_live;
2876     
2877     HashMap<Node*, LValue> m_phis;
2878     
2879     Operands<ValueSource> m_valueSources;
2880     int m_lastSetOperand;
2881     ExitThunkGenerator m_exitThunkGenerator;
2882     
2883     InPlaceAbstractState m_state;
2884     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
2885     BasicBlock* m_highBlock;
2886     BasicBlock* m_nextHighBlock;
2887     LBasicBlock m_nextLowBlock;
2888     
2889     CodeOrigin m_codeOriginForExitTarget;
2890     CodeOrigin m_codeOriginForExitProfile;
2891     unsigned m_nodeIndex;
2892     Node* m_node;
2893     SpeculationDirection m_direction;
2894 };
2895
2896 void lowerDFGToLLVM(State& state)
2897 {
2898     LowerDFGToLLVM lowering(state);
2899     lowering.lower();
2900 }
2901
2902 } } // namespace JSC::FTL
2903
2904 #endif // ENABLE(FTL_JIT)
2905