68c5efb78665b2390778f512bdf5a9106a02a54f
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToLLVM.cpp
1 /*
2  * Copyright (C) 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToLLVM.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "CodeBlockWithJITType.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGInPlaceAbstractState.h"
34 #include "FTLAbstractHeapRepository.h"
35 #include "FTLExitThunkGenerator.h"
36 #include "FTLFormattedValue.h"
37 #include "FTLLoweredNodeValue.h"
38 #include "FTLOutput.h"
39 #include "FTLThunks.h"
40 #include "FTLValueSource.h"
41 #include "LinkBuffer.h"
42 #include "OperandsInlines.h"
43 #include "Operations.h"
44 #include <wtf/ProcessID.h>
45
46 namespace JSC { namespace FTL {
47
48 using namespace DFG;
49
50 static int compileCounter;
51
52 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
53 // significantly less dead code.
54 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
55         FormattedValue _ftc_lowValue = (lowValue);                      \
56         Edge _ftc_highValue = (highValue);                              \
57         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
58         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
59             break;                                                      \
60         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
61     } while (false)
62
63 class LowerDFGToLLVM {
64 public:
65     LowerDFGToLLVM(State& state)
66         : m_graph(state.graph)
67         , m_ftlState(state)
68         , m_heaps(state.context)
69         , m_out(state.context)
70         , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
71         , m_lastSetOperand(std::numeric_limits<int>::max())
72         , m_exitThunkGenerator(state)
73         , m_state(state.graph)
74         , m_interpreter(state.graph, m_state)
75     {
76     }
77     
78     void lower()
79     {
80         CString name;
81         if (verboseCompilationEnabled()) {
82             name = toCString(
83                 "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
84                 "_", codeBlock()->hash());
85         } else
86             name = "jsBody";
87         
88         m_graph.m_dominators.computeIfNecessary(m_graph);
89         
90         m_ftlState.module =
91             LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
92         
93         m_ftlState.function = addFunction(
94             m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
95         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
96         
97         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
98         
99         m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
100         m_out.appendTo(m_prologue);
101         createPhiVariables();
102         
103         m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
104
105         m_callFrame = m_out.param(0);
106         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
107         m_tagMask = m_out.constInt64(TagMask);
108         
109         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
110             m_highBlock = m_graph.block(blockIndex);
111             if (!m_highBlock)
112                 continue;
113             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
114         }
115         
116         Vector<BasicBlock*> depthFirst;
117         m_graph.getBlocksInDepthFirstOrder(depthFirst);
118         for (unsigned i = 0; i < depthFirst.size(); ++i)
119             compileBlock(depthFirst[i]);
120         
121         // And now complete the initialization block.
122         linkOSRExitsAndCompleteInitializationBlocks();
123
124         if (Options::dumpLLVMIR())
125             dumpModule(m_ftlState.module);
126         
127         if (verboseCompilationEnabled())
128             m_ftlState.dumpState("after lowering");
129         if (validationEnabled())
130             verifyModule(m_ftlState.module);
131     }
132
133 private:
134     
135     void createPhiVariables()
136     {
137         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
138             BasicBlock* block = m_graph.block(blockIndex);
139             if (!block)
140                 continue;
141             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
142                 Node* node = block->at(nodeIndex);
143                 if (node->op() != Phi)
144                     continue;
145                 LType type;
146                 switch (node->flags() & NodeResultMask) {
147                 case NodeResultNumber:
148                     type = m_out.doubleType;
149                     break;
150                 case NodeResultInt32:
151                     type = m_out.int32;
152                     break;
153                 case NodeResultBoolean:
154                     type = m_out.boolean;
155                     break;
156                 case NodeResultJS:
157                     type = m_out.int64;
158                     break;
159                 default:
160                     RELEASE_ASSERT_NOT_REACHED();
161                     break;
162                 }
163                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
164             }
165         }
166     }
167     
168     void compileBlock(BasicBlock* block)
169     {
170         if (!block)
171             return;
172         
173         if (verboseCompilationEnabled())
174             dataLog("Compiling block ", *block, "\n");
175         
176         m_highBlock = block;
177         
178         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
179         
180         m_nextHighBlock = 0;
181         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
182             m_nextHighBlock = m_graph.block(nextBlockIndex);
183             if (m_nextHighBlock)
184                 break;
185         }
186         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
187         
188         // All of this effort to find the next block gives us the ability to keep the
189         // generated IR in roughly program order. This ought not affect the performance
190         // of the generated code (since we expect LLVM to reorder things) but it will
191         // make IR dumps easier to read.
192         m_out.appendTo(lowBlock, m_nextLowBlock);
193         
194         if (!m_highBlock->cfaHasVisited) {
195             m_out.crash();
196             return;
197         }
198         
199         initializeOSRExitStateForBlock();
200         
201         m_live = block->ssa->liveAtHead;
202         
203         m_state.reset();
204         m_state.beginBasicBlock(m_highBlock);
205         
206         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
207             if (!compileNode(m_nodeIndex))
208                 break;
209         }
210     }
211     
212     bool compileNode(unsigned nodeIndex)
213     {
214         if (!m_state.isValid()) {
215             m_out.unreachable();
216             return false;
217         }
218         
219         m_node = m_highBlock->at(nodeIndex);
220         m_codeOriginForExitProfile = m_node->codeOrigin;
221         m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
222         
223         if (verboseCompilationEnabled())
224             dataLog("Lowering ", m_node, "\n");
225         
226         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
227         
228         m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
229         
230         switch (m_node->op()) {
231         case Upsilon:
232             compileUpsilon();
233             break;
234         case Phi:
235             compilePhi();
236             break;
237         case JSConstant:
238             compileJSConstant();
239             break;
240         case WeakJSConstant:
241             compileWeakJSConstant();
242             break;
243         case GetArgument:
244             compileGetArgument();
245             break;
246         case GetLocal:
247             compileGetLocal();
248             break;
249         case SetLocal:
250             compileSetLocal();
251             break;
252         case MovHint:
253             compileMovHint();
254             break;
255         case ZombieHint:
256             compileZombieHint();
257             break;
258         case MovHintAndCheck:
259             compileMovHintAndCheck();
260             break;
261         case Phantom:
262             compilePhantom();
263             break;
264         case Flush:
265         case PhantomLocal:
266         case SetArgument:
267             break;
268         case ArithAdd:
269         case ValueAdd:
270             compileAdd();
271             break;
272         case ArithSub:
273             compileArithSub();
274             break;
275         case ArithMul:
276             compileArithMul();
277             break;
278         case ArithDiv:
279             compileArithDiv();
280             break;
281         case ArithMod:
282             compileArithMod();
283             break;
284         case ArithMin:
285         case ArithMax:
286             compileArithMinOrMax();
287             break;
288         case ArithAbs:
289             compileArithAbs();
290             break;
291         case ArithNegate:
292             compileArithNegate();
293             break;
294         case BitAnd:
295             compileBitAnd();
296             break;
297         case BitOr:
298             compileBitOr();
299             break;
300         case BitXor:
301             compileBitXor();
302             break;
303         case BitRShift:
304             compileBitRShift();
305             break;
306         case BitLShift:
307             compileBitLShift();
308             break;
309         case BitURShift:
310             compileBitURShift();
311             break;
312         case UInt32ToNumber:
313             compileUInt32ToNumber();
314             break;
315         case Int32ToDouble:
316             compileInt32ToDouble();
317             break;
318         case CheckStructure:
319             compileCheckStructure();
320             break;
321         case StructureTransitionWatchpoint:
322             compileStructureTransitionWatchpoint();
323             break;
324         case ArrayifyToStructure:
325             compileArrayifyToStructure();
326             break;
327         case PutStructure:
328             compilePutStructure();
329             break;
330         case PhantomPutStructure:
331             compilePhantomPutStructure();
332             break;
333         case GetButterfly:
334             compileGetButterfly();
335             break;
336         case GetArrayLength:
337             compileGetArrayLength();
338             break;
339         case GetByVal:
340             compileGetByVal();
341             break;
342         case PutByVal:
343         case PutByValAlias:
344             compilePutByVal();
345             break;
346         case GetByOffset:
347             compileGetByOffset();
348             break;
349         case PutByOffset:
350             compilePutByOffset();
351             break;
352         case GetGlobalVar:
353             compileGetGlobalVar();
354             break;
355         case PutGlobalVar:
356             compilePutGlobalVar();
357             break;
358         case CompareEq:
359             compileCompareEq();
360             break;
361         case CompareEqConstant:
362             compileCompareEqConstant();
363             break;
364         case CompareStrictEq:
365             compileCompareStrictEq();
366             break;
367         case CompareStrictEqConstant:
368             compileCompareStrictEqConstant();
369             break;
370         case CompareLess:
371             compileCompareLess();
372             break;
373         case CompareLessEq:
374             compileCompareLessEq();
375             break;
376         case CompareGreater:
377             compileCompareGreater();
378             break;
379         case CompareGreaterEq:
380             compileCompareGreaterEq();
381             break;
382         case LogicalNot:
383             compileLogicalNot();
384             break;
385         case Jump:
386             compileJump();
387             break;
388         case Branch:
389             compileBranch();
390             break;
391         case Switch:
392             compileSwitch();
393             break;
394         case Return:
395             compileReturn();
396             break;
397         case ForceOSRExit:
398             compileForceOSRExit();
399             break;
400         default:
401             RELEASE_ASSERT_NOT_REACHED();
402             break;
403         }
404         
405         if (m_node->shouldGenerate())
406             DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
407         
408         if (m_node->adjustedRefCount())
409             m_live.add(m_node);
410         
411         if (shouldExecuteEffects)
412             m_interpreter.executeEffects(nodeIndex);
413         
414         return true;
415     }
416     
417     void compileUpsilon()
418     {
419         LValue destination = m_phis.get(m_node->phi());
420         
421         switch (m_node->child1().useKind()) {
422         case NumberUse:
423             m_out.set(lowDouble(m_node->child1()), destination);
424             break;
425         case Int32Use:
426             m_out.set(lowInt32(m_node->child1()), destination);
427             break;
428         case BooleanUse:
429             m_out.set(lowBoolean(m_node->child1()), destination);
430             break;
431         case CellUse:
432             m_out.set(lowCell(m_node->child1()), destination);
433             break;
434         case UntypedUse:
435             m_out.set(lowJSValue(m_node->child1()), destination);
436             break;
437         default:
438             RELEASE_ASSERT_NOT_REACHED();
439             break;
440         }
441     }
442     
443     void compilePhi()
444     {
445         LValue source = m_phis.get(m_node);
446         
447         switch (m_node->flags() & NodeResultMask) {
448         case NodeResultNumber:
449             setDouble(m_out.get(source));
450             break;
451         case NodeResultInt32:
452             setInt32(m_out.get(source));
453             break;
454         case NodeResultBoolean:
455             setBoolean(m_out.get(source));
456             break;
457         case NodeResultJS:
458             setJSValue(m_out.get(source));
459             break;
460         default:
461             RELEASE_ASSERT_NOT_REACHED();
462             break;
463         }
464     }
465     
466     void compileJSConstant()
467     {
468         JSValue value = m_graph.valueOfJSConstant(m_node);
469         if (value.isDouble())
470             setDouble(m_out.constDouble(value.asDouble()));
471         else
472             setJSValue(m_out.constInt64(JSValue::encode(value)));
473     }
474     
475     void compileWeakJSConstant()
476     {
477         setJSValue(weakPointer(m_node->weakConstant()));
478     }
479     
480     void compileGetArgument()
481     {
482         VariableAccessData* variable = m_node->variableAccessData();
483         int operand = variable->operand();
484
485         LValue jsValue = m_out.load64(addressFor(operand));
486
487         switch (useKindFor(variable->flushFormat())) {
488         case Int32Use:
489             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
490             setInt32(unboxInt32(jsValue));
491             break;
492         case CellUse:
493             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
494             setJSValue(jsValue);
495             break;
496         case BooleanUse:
497             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
498             setBoolean(unboxBoolean(jsValue));
499             break;
500         case UntypedUse:
501             setJSValue(jsValue);
502             break;
503         default:
504             RELEASE_ASSERT_NOT_REACHED();
505             break;
506         }
507     }
508     
509     void compileGetLocal()
510     {
511         // GetLocals arise only for captured variables.
512         
513         VariableAccessData* variable = m_node->variableAccessData();
514         AbstractValue& value = m_state.variables().operand(variable->local());
515         
516         RELEASE_ASSERT(variable->isCaptured());
517         
518         if (isInt32Speculation(value.m_value))
519             setInt32(m_out.load32(payloadFor(variable->local())));
520         else
521             setJSValue(m_out.load64(addressFor(variable->local())));
522     }
523     
524     void compileSetLocal()
525     {
526         observeMovHint(m_node);
527         
528         VariableAccessData* variable = m_node->variableAccessData();
529         SpeculatedType prediction = variable->argumentAwarePrediction();
530         
531         if (variable->shouldUnboxIfPossible()) {
532             if (variable->shouldUseDoubleFormat()) {
533                 LValue value = lowDouble(m_node->child1());
534                 m_out.storeDouble(value, addressFor(variable->local()));
535                 m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
536                 return;
537             }
538             
539             if (isInt32Speculation(prediction)) {
540                 LValue value = lowInt32(m_node->child1());
541                 m_out.store32(value, payloadFor(variable->local()));
542                 m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
543                 return;
544             }
545             if (isCellSpeculation(prediction)) {
546                 LValue value = lowCell(m_node->child1());
547                 m_out.store64(value, addressFor(variable->local()));
548                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
549                 return;
550             }
551             if (isBooleanSpeculation(prediction)) {
552                 speculateBoolean(m_node->child1());
553                 m_out.store64(
554                     lowJSValue(m_node->child1(), ManualOperandSpeculation),
555                     addressFor(variable->local()));
556                 m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
557                 return;
558             }
559         }
560         
561         LValue value = lowJSValue(m_node->child1());
562         m_out.store64(value, addressFor(variable->local()));
563         m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
564     }
565     
566     void compileMovHint()
567     {
568         observeMovHint(m_node);
569     }
570     
571     void compileZombieHint()
572     {
573         VariableAccessData* data = m_node->variableAccessData();
574         m_lastSetOperand = data->local();
575         m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
576     }
577     
578     void compileMovHintAndCheck()
579     {
580         observeMovHint(m_node);
581         speculate(m_node->child1());
582     }
583     
584     void compilePhantom()
585     {
586         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
587     }
588     
589     void compileAdd()
590     {
591         switch (m_node->binaryUseKind()) {
592         case Int32Use: {
593             LValue left = lowInt32(m_node->child1());
594             LValue right = lowInt32(m_node->child2());
595             
596             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
597                 setInt32(m_out.add(left, right));
598                 break;
599             }
600             
601             LValue result = m_out.addWithOverflow32(left, right);
602             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
603             setInt32(m_out.extractValue(result, 0));
604             break;
605         }
606             
607         case NumberUse: {
608             setDouble(
609                 m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
610             break;
611         }
612             
613         default:
614             RELEASE_ASSERT_NOT_REACHED();
615             break;
616         }
617     }
618     
619     void compileArithSub()
620     {
621         switch (m_node->binaryUseKind()) {
622         case Int32Use: {
623             LValue left = lowInt32(m_node->child1());
624             LValue right = lowInt32(m_node->child2());
625             
626             if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
627                 setInt32(m_out.sub(left, right));
628                 break;
629             }
630             
631             LValue result = m_out.subWithOverflow32(left, right);
632             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
633             setInt32(m_out.extractValue(result, 0));
634             break;
635         }
636             
637         case NumberUse: {
638             setDouble(
639                 m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
640             break;
641         }
642             
643         default:
644             RELEASE_ASSERT_NOT_REACHED();
645             break;
646         }
647     }
648     
649     void compileArithMul()
650     {
651         switch (m_node->binaryUseKind()) {
652         case Int32Use: {
653             LValue left = lowInt32(m_node->child1());
654             LValue right = lowInt32(m_node->child2());
655             
656             LValue result;
657             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
658                 result = m_out.mul(left, right);
659             else {
660                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
661                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
662                 result = m_out.extractValue(overflowResult, 0);
663             }
664             
665             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
666                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
667                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
668                 
669                 m_out.branch(m_out.notZero32(result), continuation, slowCase);
670                 
671                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
672                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
673                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
674                 m_out.jump(continuation);
675                 m_out.appendTo(continuation, lastNext);
676             }
677             
678             setInt32(result);
679             break;
680         }
681             
682         case NumberUse: {
683             setDouble(
684                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
685             break;
686         }
687             
688         default:
689             RELEASE_ASSERT_NOT_REACHED();
690             break;
691         }
692     }
693     
694     void compileArithDiv()
695     {
696         switch (m_node->binaryUseKind()) {
697         case Int32Use: {
698             LValue numerator = lowInt32(m_node->child1());
699             LValue denominator = lowInt32(m_node->child2());
700             
701             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
702             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
703             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
704             
705             Vector<ValueFromBlock, 3> results;
706             
707             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
708             
709             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
710             
711             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
712             
713             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
714             
715             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
716                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
717                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
718                 m_out.jump(continuation);
719             } else {
720                 // This is the case where we convert the result to an int after we're done. So,
721                 // if the denominator is zero, then the result should be result should be zero.
722                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
723                 // check above) and the numerator is -2^31 then the result should be -2^31.
724                 
725                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
726                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
727                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
728                 
729                 m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
730                 
731                 m_out.appendTo(divByZero, notDivByZero);
732                 results.append(m_out.anchor(m_out.int32Zero));
733                 m_out.jump(done);
734                 
735                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
736                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
737                 
738                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
739                 results.append(m_out.anchor(neg2ToThe31));
740                 m_out.jump(done);
741             }
742             
743             m_out.appendTo(continuation, done);
744             
745             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
746                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
747                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
748                 
749                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
750                 
751                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
752                 
753                 speculate(
754                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
755                 
756                 m_out.jump(numeratorContinuation);
757                 
758                 m_out.appendTo(numeratorContinuation, innerLastNext);
759             }
760             
761             LValue divisionResult = m_out.div(numerator, denominator);
762             
763             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
764                 speculate(
765                     Overflow, noValue(), 0,
766                     m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
767             }
768             
769             results.append(m_out.anchor(divisionResult));
770             m_out.jump(done);
771             
772             m_out.appendTo(done, lastNext);
773             
774             setInt32(m_out.phi(m_out.int32, results));
775             break;
776         }
777             
778         case NumberUse: {
779             setDouble(
780                 m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
781             break;
782         }
783             
784         default:
785             RELEASE_ASSERT_NOT_REACHED();
786             break;
787         }
788     }
789     
790     void compileArithMod()
791     {
792         switch (m_node->binaryUseKind()) {
793         case Int32Use: {
794             LValue numerator = lowInt32(m_node->child1());
795             LValue denominator = lowInt32(m_node->child2());
796             
797             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
798             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
799             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
800             
801             Vector<ValueFromBlock, 3> results;
802             
803             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
804             
805             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
806             
807             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
808             
809             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
810             
811             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
812             // separate case for that. But it probably doesn't matter so much.
813             if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
814                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
815                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
816                 m_out.jump(continuation);
817             } else {
818                 // This is the case where we convert the result to an int after we're done. So,
819                 // if the denominator is zero, then the result should be result should be zero.
820                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
821                 // check above) and the numerator is -2^31 then the result should be -2^31.
822                 
823                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
824                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
825                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
826                 
827                 m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
828                 
829                 m_out.appendTo(modByZero, notModByZero);
830                 results.append(m_out.anchor(m_out.int32Zero));
831                 m_out.jump(done);
832                 
833                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
834                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
835                 
836                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
837                 results.append(m_out.anchor(m_out.int32Zero));
838                 m_out.jump(done);
839             }
840             
841             m_out.appendTo(continuation, done);
842             
843             if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
844                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod zero numerator"));
845                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
846                 
847                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
848                 
849                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
850                 
851                 speculate(
852                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
853                 
854                 m_out.jump(numeratorContinuation);
855                 
856                 m_out.appendTo(numeratorContinuation, innerLastNext);
857             }
858             
859             results.append(m_out.anchor(m_out.rem(numerator, denominator)));
860             m_out.jump(done);
861             
862             m_out.appendTo(done, lastNext);
863             
864             setInt32(m_out.phi(m_out.int32, results));
865             break;
866         }
867             
868         case NumberUse: {
869             setDouble(
870                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
871             break;
872         }
873             
874         default:
875             RELEASE_ASSERT_NOT_REACHED();
876             break;
877         }
878     }
879
880     void compileArithMinOrMax()
881     {
882         switch (m_node->binaryUseKind()) {
883         case Int32Use: {
884             LValue left = lowInt32(m_node->child1());
885             LValue right = lowInt32(m_node->child2());
886             
887             setInt32(
888                 m_out.select(
889                     m_node->op() == ArithMin
890                         ? m_out.lessThan(left, right)
891                         : m_out.lessThan(right, left),
892                     left, right));
893             break;
894         }
895             
896         case NumberUse: {
897             LValue left = lowDouble(m_node->child1());
898             LValue right = lowDouble(m_node->child2());
899             
900             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
901             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
902             
903             Vector<ValueFromBlock, 2> results;
904             
905             results.append(m_out.anchor(left));
906             m_out.branch(
907                 m_node->op() == ArithMin
908                     ? m_out.doubleLessThan(left, right)
909                     : m_out.doubleGreaterThan(left, right),
910                 continuation, notLessThan);
911             
912             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
913             results.append(m_out.anchor(m_out.select(
914                 m_node->op() == ArithMin
915                     ? m_out.doubleGreaterThanOrEqual(left, right)
916                     : m_out.doubleLessThanOrEqual(left, right),
917                 right, m_out.constDouble(0.0 / 0.0))));
918             m_out.jump(continuation);
919             
920             m_out.appendTo(continuation, lastNext);
921             setDouble(m_out.phi(m_out.doubleType, results));
922             break;
923         }
924             
925         default:
926             RELEASE_ASSERT_NOT_REACHED();
927             break;
928         }
929     }
930     
931     void compileArithAbs()
932     {
933         switch (m_node->child1().useKind()) {
934         case Int32Use: {
935             LValue value = lowInt32(m_node->child1());
936             
937             LValue mask = m_out.aShr(value, m_out.constInt32(31));
938             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
939             
940             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
941             
942             setInt32(result);
943             break;
944         }
945             
946         case NumberUse: {
947             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
948             break;
949         }
950             
951         default:
952             RELEASE_ASSERT_NOT_REACHED();
953             break;
954         }
955     }
956     
957     void compileArithNegate()
958     {
959         switch (m_node->child1().useKind()) {
960         case Int32Use: {
961             LValue value = lowInt32(m_node->child1());
962             
963             LValue result;
964             if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
965                 result = m_out.neg(value);
966             else {
967                 // We don't have a negate-with-overflow intrinsic. Hopefully this
968                 // does the trick, though.
969                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
970                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
971                 result = m_out.extractValue(overflowResult, 0);
972             }
973             
974             setInt32(result);
975             break;
976         }
977             
978         case NumberUse: {
979             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
980             break;
981         }
982             
983         default:
984             RELEASE_ASSERT_NOT_REACHED();
985             break;
986         }
987     }
988     
989     void compileBitAnd()
990     {
991         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
992     }
993     
994     void compileBitOr()
995     {
996         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
997     }
998     
999     void compileBitXor()
1000     {
1001         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1002     }
1003     
1004     void compileBitRShift()
1005     {
1006         setInt32(m_out.aShr(
1007             lowInt32(m_node->child1()),
1008             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1009     }
1010     
1011     void compileBitLShift()
1012     {
1013         setInt32(m_out.shl(
1014             lowInt32(m_node->child1()),
1015             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1016     }
1017     
1018     void compileBitURShift()
1019     {
1020         setInt32(m_out.lShr(
1021             lowInt32(m_node->child1()),
1022             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1023     }
1024     
1025     void compileUInt32ToNumber()
1026     {
1027         LValue value = lowInt32(m_node->child1());
1028
1029         if (!nodeCanSpeculateInteger(m_node->arithNodeFlags())) {
1030             setDouble(m_out.unsignedToDouble(value));
1031             return;
1032         }
1033         
1034         speculateForward(
1035             Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
1036             FormattedValue(ValueFormatUInt32, value));
1037         setInt32(value);
1038     }
1039     
1040     void compileInt32ToDouble()
1041     {
1042         // This node is tricky to compile in the DFG backend because it tries to
1043         // avoid converting child1 to a double in-place, as that would make subsequent
1044         // int uses of of child1 fail. But the FTL needs no such special magic, since
1045         // unlike the DFG backend, the FTL allows each node to have multiple
1046         // contemporaneous low-level representations. So, this gives child1 a double
1047         // representation and then forwards that representation to m_node.
1048         
1049         setDouble(lowDouble(m_node->child1()));
1050     }
1051     
1052     void compileCheckStructure()
1053     {
1054         LValue cell = lowCell(m_node->child1());
1055         
1056         ExitKind exitKind;
1057         if (m_node->child1()->op() == WeakJSConstant)
1058             exitKind = BadWeakConstantCache;
1059         else
1060             exitKind = BadCache;
1061         
1062         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1063         
1064         if (m_node->structureSet().size() == 1) {
1065             speculate(
1066                 exitKind, jsValueValue(cell), 0,
1067                 m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
1068             return;
1069         }
1070         
1071         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1072         
1073         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1074         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1075             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1076             m_out.branch(
1077                 m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
1078                 continuation, nextStructure);
1079             m_out.appendTo(nextStructure);
1080         }
1081         
1082         speculate(
1083             exitKind, jsValueValue(cell), 0,
1084             m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
1085         
1086         m_out.jump(continuation);
1087         m_out.appendTo(continuation, lastNext);
1088     }
1089     
1090     void compileStructureTransitionWatchpoint()
1091     {
1092         addWeakReference(m_node->structure());
1093         
1094         // FIXME: Implement structure transition watchpoints.
1095         // https://bugs.webkit.org/show_bug.cgi?id=113647
1096         
1097         speculateCell(m_node->child1());
1098     }
1099     
1100     void compileArrayifyToStructure()
1101     {
1102         LValue cell = lowCell(m_node->child1());
1103         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1104         
1105         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1106         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1107         
1108         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1109         
1110         m_out.branch(
1111             m_out.notEqual(structure, weakPointer(m_node->structure())),
1112             unexpectedStructure, continuation);
1113         
1114         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1115         
1116         if (property) {
1117             switch (m_node->arrayMode().type()) {
1118             case Array::Int32:
1119             case Array::Double:
1120             case Array::Contiguous:
1121                 speculate(
1122                     Uncountable, noValue(), 0,
1123                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1124                 break;
1125             default:
1126                 break;
1127             }
1128         }
1129         
1130         switch (m_node->arrayMode().type()) {
1131         case Array::Int32:
1132             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1133             break;
1134         case Array::Double:
1135             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1136             break;
1137         case Array::Contiguous:
1138             if (m_node->arrayMode().conversion() == Array::RageConvert)
1139                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1140             else
1141                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1142             break;
1143         case Array::ArrayStorage:
1144         case Array::SlowPutArrayStorage:
1145             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1146             break;
1147         default:
1148             RELEASE_ASSERT_NOT_REACHED();
1149             break;
1150         }
1151         
1152         structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1153         speculate(
1154             BadIndexingType, jsValueValue(cell), 0,
1155             m_out.notEqual(structure, weakPointer(m_node->structure())));
1156         m_out.jump(continuation);
1157         
1158         m_out.appendTo(continuation, lastNext);
1159     }
1160     
1161     void compilePutStructure()
1162     {
1163         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1164         
1165         m_out.store64(
1166             m_out.constIntPtr(m_node->structureTransitionData().newStructure),
1167             lowCell(m_node->child1()), m_heaps.JSCell_structure);
1168     }
1169     
1170     void compilePhantomPutStructure()
1171     {
1172         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1173     }
1174     
1175     void compileGetButterfly()
1176     {
1177         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1178     }
1179     
1180     void compileGetArrayLength()
1181     {
1182         switch (m_node->arrayMode().type()) {
1183         case Array::Int32:
1184         case Array::Double:
1185         case Array::Contiguous: {
1186             setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1187             break;
1188         }
1189             
1190         default:
1191             RELEASE_ASSERT_NOT_REACHED();
1192             break;
1193         }
1194     }
1195     
1196     void compileGetByVal()
1197     {
1198         LValue index = lowInt32(m_node->child2());
1199         LValue storage = lowStorage(m_node->child3());
1200         
1201         switch (m_node->arrayMode().type()) {
1202         case Array::Int32:
1203         case Array::Contiguous: {
1204             if (m_node->arrayMode().isInBounds()) {
1205                 speculate(
1206                     OutOfBounds, noValue(), 0,
1207                     m_out.aboveOrEqual(
1208                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1209                 
1210                 LValue result = m_out.load64(m_out.baseIndex(
1211                     m_node->arrayMode().type() == Array::Int32 ?
1212                         m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1213                     storage, m_out.zeroExt(index, m_out.intPtr),
1214                     m_state.forNode(m_node->child2()).m_value));
1215                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1216                 setJSValue(result);
1217                 return;
1218             }
1219             
1220             // FIXME: Implement hole/OOB loads in the FTL.
1221             // https://bugs.webkit.org/show_bug.cgi?id=118077
1222             RELEASE_ASSERT_NOT_REACHED();
1223             return;
1224         }
1225             
1226         case Array::Double: {
1227             if (m_node->arrayMode().isInBounds()) {
1228                 if (m_node->arrayMode().isSaneChain()) {
1229                     // FIXME: Implement structure transition watchpoints.
1230                     // https://bugs.webkit.org/show_bug.cgi?id=113647
1231                 }
1232             
1233                 speculate(
1234                     OutOfBounds, noValue(), 0,
1235                     m_out.aboveOrEqual(
1236                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1237                 
1238                 LValue result = m_out.loadDouble(m_out.baseIndex(
1239                     m_heaps.indexedDoubleProperties,
1240                     storage, m_out.zeroExt(index, m_out.intPtr),
1241                     m_state.forNode(m_node->child2()).m_value));
1242                 
1243                 if (!m_node->arrayMode().isSaneChain()) {
1244                     speculate(
1245                         LoadFromHole, noValue(), 0,
1246                         m_out.doubleNotEqualOrUnordered(result, result));
1247                 }
1248                 setDouble(result);
1249                 break;
1250             }
1251             
1252             // FIXME: Implement hole/OOB loads in the FTL.
1253             // https://bugs.webkit.org/show_bug.cgi?id=118077
1254             RELEASE_ASSERT_NOT_REACHED();
1255             return;
1256         }
1257             
1258         default:
1259             RELEASE_ASSERT_NOT_REACHED();
1260             return;
1261         }
1262     }
1263     
1264     void compilePutByVal()
1265     {
1266         Edge child1 = m_graph.varArgChild(m_node, 0);
1267         Edge child2 = m_graph.varArgChild(m_node, 1);
1268         Edge child3 = m_graph.varArgChild(m_node, 2);
1269         Edge child4 = m_graph.varArgChild(m_node, 3);
1270
1271         LValue base = lowCell(child1);
1272         LValue index = lowInt32(child2);
1273         LValue storage = lowStorage(child4);
1274         
1275         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
1276         LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
1277             
1278         switch (m_node->arrayMode().type()) {
1279         case Array::Int32:
1280         case Array::Contiguous: {
1281             LValue value = lowJSValue(child3, ManualOperandSpeculation);
1282             
1283             if (m_node->arrayMode().type() == Array::Int32)
1284                 FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
1285             
1286             TypedPointer elementPointer = m_out.baseIndex(
1287                 m_node->arrayMode().type() == Array::Int32 ?
1288                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1289                 storage, m_out.zeroExt(index, m_out.intPtr),
1290                 m_state.forNode(child2).m_value);
1291             
1292             if (m_node->op() == PutByValAlias) {
1293                 m_out.store64(value, elementPointer);
1294                 break;
1295             }
1296             
1297             contiguousPutByValOutOfBounds(
1298                 codeBlock()->isStrictMode()
1299                     ? operationPutByValBeyondArrayBoundsStrict
1300                     : operationPutByValBeyondArrayBoundsNonStrict,
1301                 base, storage, index, value, continuation);
1302             
1303             m_out.store64(value, elementPointer);
1304             break;
1305         }
1306             
1307         case Array::Double: {
1308             LValue value = lowDouble(child3);
1309             
1310             FTL_TYPE_CHECK(
1311                 doubleValue(value), child3, SpecRealNumber,
1312                 m_out.doubleNotEqualOrUnordered(value, value));
1313             
1314             TypedPointer elementPointer = m_out.baseIndex(
1315                 m_heaps.indexedDoubleProperties,
1316                 storage, m_out.zeroExt(index, m_out.intPtr),
1317                 m_state.forNode(child2).m_value);
1318
1319             if (m_node->op() == PutByValAlias) {
1320                 m_out.storeDouble(value, elementPointer);
1321                 break;
1322             }
1323             
1324             contiguousPutByValOutOfBounds(
1325                 codeBlock()->isStrictMode()
1326                     ? operationPutDoubleByValBeyondArrayBoundsStrict
1327                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1328                 base, storage, index, value, continuation);
1329             
1330             m_out.storeDouble(value, elementPointer);
1331             break;
1332         }
1333             
1334         default:
1335             RELEASE_ASSERT_NOT_REACHED();
1336             break;
1337         }
1338
1339         m_out.jump(continuation);
1340         m_out.appendTo(continuation, outerLastNext);
1341     }
1342     
1343     void compileGetByOffset()
1344     {
1345         StorageAccessData& data =
1346             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1347         
1348         setJSValue(
1349             m_out.load64(
1350                 m_out.address(
1351                     m_heaps.properties[data.identifierNumber],
1352                     lowStorage(m_node->child1()),
1353                     offsetRelativeToBase(data.offset))));
1354     }
1355     
1356     void compilePutByOffset()
1357     {
1358         StorageAccessData& data =
1359             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1360         
1361         m_out.store64(
1362             lowJSValue(m_node->child3()),
1363             m_out.address(
1364                 m_heaps.properties[data.identifierNumber],
1365                 lowStorage(m_node->child1()),
1366                 offsetRelativeToBase(data.offset)));
1367     }
1368     
1369     void compileGetGlobalVar()
1370     {
1371         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
1372     }
1373     
1374     void compilePutGlobalVar()
1375     {
1376         m_out.store64(
1377             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
1378     }
1379     
1380     void compileCompareEq()
1381     {
1382         if (m_node->isBinaryUseKind(Int32Use)
1383             || m_node->isBinaryUseKind(NumberUse)
1384             || m_node->isBinaryUseKind(ObjectUse)) {
1385             compileCompareStrictEq();
1386             return;
1387         }
1388         
1389         RELEASE_ASSERT_NOT_REACHED();
1390     }
1391     
1392     void compileCompareEqConstant()
1393     {
1394         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
1395         masqueradesAsUndefinedWatchpointIfIsStillValid();
1396         setBoolean(
1397             equalNullOrUndefined(
1398                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
1399     }
1400     
1401     void compileCompareStrictEq()
1402     {
1403         if (m_node->isBinaryUseKind(Int32Use)) {
1404             setBoolean(
1405                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1406             return;
1407         }
1408         
1409         if (m_node->isBinaryUseKind(NumberUse)) {
1410             setBoolean(
1411                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1412         }
1413         
1414         if (m_node->isBinaryUseKind(ObjectUse)) {
1415             masqueradesAsUndefinedWatchpointIfIsStillValid();
1416             setBoolean(
1417                 m_out.equal(
1418                     lowNonNullObject(m_node->child1()),
1419                     lowNonNullObject(m_node->child2())));
1420             return;
1421         }
1422         
1423         RELEASE_ASSERT_NOT_REACHED();
1424     }
1425     
1426     void compileCompareStrictEqConstant()
1427     {
1428         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
1429
1430         if (constant.isUndefinedOrNull()
1431             && !masqueradesAsUndefinedWatchpointIfIsStillValid()) {
1432             if (constant.isNull()) {
1433                 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull));
1434                 return;
1435             }
1436         
1437             ASSERT(constant.isUndefined());
1438             setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
1439             return;
1440         }
1441         
1442         setBoolean(
1443             m_out.equal(
1444                 lowJSValue(m_node->child1()),
1445                 m_out.constInt64(JSValue::encode(constant))));
1446     }
1447     
1448     void compileCompareLess()
1449     {
1450         if (m_node->isBinaryUseKind(Int32Use)) {
1451             setBoolean(
1452                 m_out.lessThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1453             return;
1454         }
1455         
1456         if (m_node->isBinaryUseKind(NumberUse)) {
1457             setBoolean(
1458                 m_out.doubleLessThan(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1459             return;
1460         }
1461         
1462         RELEASE_ASSERT_NOT_REACHED();
1463     }
1464     
1465     void compileCompareLessEq()
1466     {
1467         if (m_node->isBinaryUseKind(Int32Use)) {
1468             setBoolean(
1469                 m_out.lessThanOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1470             return;
1471         }
1472         
1473         if (m_node->isBinaryUseKind(NumberUse)) {
1474             setBoolean(
1475                 m_out.doubleLessThanOrEqual(
1476                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1477             return;
1478         }
1479         
1480         RELEASE_ASSERT_NOT_REACHED();
1481     }
1482     
1483     void compileCompareGreater()
1484     {
1485         if (m_node->isBinaryUseKind(Int32Use)) {
1486             setBoolean(
1487                 m_out.greaterThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1488             return;
1489         }
1490         
1491         if (m_node->isBinaryUseKind(NumberUse)) {
1492             setBoolean(
1493                 m_out.doubleGreaterThan(
1494                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1495             return;
1496         }
1497         
1498         RELEASE_ASSERT_NOT_REACHED();
1499     }
1500     
1501     void compileCompareGreaterEq()
1502     {
1503         if (m_node->isBinaryUseKind(Int32Use)) {
1504             setBoolean(
1505                 m_out.greaterThanOrEqual(
1506                     lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1507             return;
1508         }
1509         
1510         if (m_node->isBinaryUseKind(NumberUse)) {
1511             setBoolean(
1512                 m_out.doubleGreaterThanOrEqual(
1513                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1514             return;
1515         }
1516         
1517         RELEASE_ASSERT_NOT_REACHED();
1518     }
1519     
1520     void compileLogicalNot()
1521     {
1522         setBoolean(m_out.bitNot(boolify(m_node->child1())));
1523     }
1524     
1525     void compileJump()
1526     {
1527         m_out.jump(lowBlock(m_node->takenBlock()));
1528     }
1529     
1530     void compileBranch()
1531     {
1532         m_out.branch(
1533             boolify(m_node->child1()),
1534             lowBlock(m_node->takenBlock()),
1535             lowBlock(m_node->notTakenBlock()));
1536     }
1537     
1538     void compileSwitch()
1539     {
1540         SwitchData* data = m_node->switchData();
1541         switch (data->kind) {
1542         case SwitchImm: {
1543             Vector<ValueFromBlock, 2> intValues;
1544             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
1545             
1546             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
1547             
1548             switch (m_node->child1().useKind()) {
1549             case Int32Use: {
1550                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
1551                 m_out.jump(switchOnInts);
1552                 break;
1553             }
1554                 
1555             case UntypedUse: {
1556                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
1557                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
1558                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
1559                 
1560                 LValue boxedValue = lowJSValue(m_node->child1());
1561                 m_out.branch(isNotInt32(boxedValue), isNotInt, isInt);
1562                 
1563                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
1564                 
1565                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
1566                 m_out.jump(switchOnInts);
1567                 
1568                 m_out.appendTo(isNotInt, isDouble);
1569                 m_out.branch(
1570                     isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble);
1571                 
1572                 m_out.appendTo(isDouble, innerLastNext);
1573                 LValue doubleValue = unboxDouble(boxedValue);
1574                 LValue intInDouble = m_out.fpToInt32(doubleValue);
1575                 intValues.append(m_out.anchor(intInDouble));
1576                 m_out.branch(
1577                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
1578                     switchOnInts, lowBlock(data->fallThrough));
1579                 break;
1580             }
1581                 
1582             default:
1583                 RELEASE_ASSERT_NOT_REACHED();
1584                 break;
1585             }
1586             
1587             m_out.appendTo(switchOnInts, lastNext);
1588             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
1589             return;
1590         }
1591         
1592         case SwitchChar: {
1593             LValue stringValue;
1594             
1595             switch (m_node->child1().useKind()) {
1596             case StringUse: {
1597                 stringValue = lowString(m_node->child1());
1598                 break;
1599             }
1600                 
1601             case UntypedUse: {
1602                 LValue unboxedValue = lowJSValue(m_node->child1());
1603                 
1604                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
1605                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
1606                 
1607                 m_out.branch(
1608                     isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase);
1609                 
1610                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
1611                 LValue cellValue = unboxedValue;
1612                 m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase);
1613                 
1614                 m_out.appendTo(isStringCase, lastNext);
1615                 stringValue = cellValue;
1616                 break;
1617             }
1618                 
1619             default:
1620                 RELEASE_ASSERT_NOT_REACHED();
1621                 break;
1622             }
1623             
1624             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
1625             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
1626             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
1627             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
1628             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
1629             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
1630             
1631             m_out.branch(
1632                 m_out.notEqual(
1633                     m_out.load32(stringValue, m_heaps.JSString_length),
1634                     m_out.int32One),
1635                 lowBlock(data->fallThrough), lengthIs1);
1636             
1637             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
1638             Vector<ValueFromBlock, 2> values;
1639             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
1640             values.append(m_out.anchor(fastValue));
1641             m_out.branch(m_out.isNull(fastValue), needResolution, resolved);
1642             
1643             m_out.appendTo(needResolution, resolved);
1644             values.append(m_out.anchor(
1645                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
1646             m_out.jump(resolved);
1647             
1648             m_out.appendTo(resolved, is8Bit);
1649             LValue value = m_out.phi(m_out.intPtr, values);
1650             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
1651             m_out.branch(
1652                 m_out.testNonZero32(
1653                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
1654                     m_out.constInt32(StringImpl::flagIs8Bit())),
1655                 is8Bit, is16Bit);
1656             
1657             Vector<ValueFromBlock, 2> characters;
1658             m_out.appendTo(is8Bit, is16Bit);
1659             characters.append(m_out.anchor(
1660                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
1661             m_out.jump(continuation);
1662             
1663             m_out.appendTo(is16Bit, continuation);
1664             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
1665             m_out.jump(continuation);
1666             
1667             m_out.appendTo(continuation, lastNext);
1668             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
1669             return;
1670         }
1671         
1672         case SwitchString:
1673             RELEASE_ASSERT_NOT_REACHED();
1674             break;
1675         }
1676         
1677         RELEASE_ASSERT_NOT_REACHED();
1678     }
1679     
1680     void compileReturn()
1681     {
1682         // FIXME: have a real epilogue when we switch to using our calling convention.
1683         // https://bugs.webkit.org/show_bug.cgi?id=113621
1684         m_out.ret(lowJSValue(m_node->child1()));
1685     }
1686     
1687     void compileForceOSRExit()
1688     {
1689         terminate(InadequateCoverage);
1690     }
1691     
1692     LValue boolify(Edge edge)
1693     {
1694         switch (edge.useKind()) {
1695         case BooleanUse:
1696             return lowBoolean(m_node->child1());
1697         case Int32Use:
1698             return m_out.notZero32(lowInt32(m_node->child1()));
1699         case NumberUse:
1700             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
1701         case ObjectOrOtherUse:
1702             return m_out.bitNot(
1703                 equalNullOrUndefined(
1704                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
1705                     ManualOperandSpeculation));
1706         default:
1707             RELEASE_ASSERT_NOT_REACHED();
1708             return 0;
1709         }
1710     }
1711     
1712     enum StringOrObjectMode {
1713         AllCellsAreFalse,
1714         CellCaseSpeculatesObject
1715     };
1716     enum EqualNullOrUndefinedMode {
1717         EqualNull,
1718         EqualUndefined,
1719         EqualNullOrUndefined,
1720         SpeculateNullOrUndefined
1721     };
1722     LValue equalNullOrUndefined(
1723         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
1724         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
1725     {
1726         bool validWatchpoint = masqueradesAsUndefinedWatchpointIfIsStillValid();
1727         
1728         LValue value = lowJSValue(edge, operandMode);
1729         
1730         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
1731         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
1732         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
1733         
1734         m_out.branch(isNotCell(value), primitiveCase, cellCase);
1735         
1736         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
1737         
1738         Vector<ValueFromBlock, 3> results;
1739         
1740         switch (cellMode) {
1741         case AllCellsAreFalse:
1742             break;
1743         case CellCaseSpeculatesObject:
1744             FTL_TYPE_CHECK(
1745                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
1746                 m_out.equal(
1747                     m_out.loadPtr(value, m_heaps.JSCell_structure),
1748                     m_out.constIntPtr(vm().stringStructure.get())));
1749             break;
1750         }
1751         
1752         if (validWatchpoint) {
1753             results.append(m_out.anchor(m_out.booleanFalse));
1754             m_out.jump(continuation);
1755         } else {
1756             LBasicBlock masqueradesCase =
1757                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
1758                 
1759             LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure);
1760             
1761             results.append(m_out.anchor(m_out.booleanFalse));
1762             
1763             m_out.branch(
1764                 m_out.testNonZero8(
1765                     m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
1766                     m_out.constInt8(MasqueradesAsUndefined)),
1767                 masqueradesCase, continuation);
1768             
1769             m_out.appendTo(masqueradesCase, primitiveCase);
1770             
1771             results.append(m_out.anchor(
1772                 m_out.equal(
1773                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)),
1774                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
1775             m_out.jump(continuation);
1776         }
1777         
1778         m_out.appendTo(primitiveCase, continuation);
1779         
1780         LValue primitiveResult;
1781         switch (primitiveMode) {
1782         case EqualNull:
1783             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
1784             break;
1785         case EqualUndefined:
1786             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
1787             break;
1788         case EqualNullOrUndefined:
1789             primitiveResult = m_out.equal(
1790                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1791                 m_out.constInt64(ValueNull));
1792             break;
1793         case SpeculateNullOrUndefined:
1794             FTL_TYPE_CHECK(
1795                 jsValueValue(value), edge, SpecCell | SpecOther,
1796                 m_out.notEqual(
1797                     m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
1798                     m_out.constInt64(ValueNull)));
1799             primitiveResult = m_out.booleanTrue;
1800             break;
1801         }
1802         results.append(m_out.anchor(primitiveResult));
1803         m_out.jump(continuation);
1804         
1805         m_out.appendTo(continuation, lastNext);
1806         
1807         return m_out.phi(m_out.boolean, results);
1808     }
1809     
1810     template<typename FunctionType>
1811     void contiguousPutByValOutOfBounds(
1812         FunctionType slowPathFunction,
1813         LValue base, LValue storage, LValue index, LValue value,
1814         LBasicBlock continuation)
1815     {
1816         LValue isNotInBounds = m_out.aboveOrEqual(
1817             index, m_out.load32(storage, m_heaps.Butterfly_publicLength));
1818         if (m_node->arrayMode().isInBounds())
1819             speculate(StoreToHoleOrOutOfBounds, noValue(), 0, isNotInBounds);
1820         else {
1821             LBasicBlock notInBoundsCase =
1822                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
1823             LBasicBlock performStore =
1824                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
1825                 
1826             m_out.branch(isNotInBounds, notInBoundsCase, performStore);
1827                 
1828             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
1829                 
1830             LValue isOutOfBounds = m_out.aboveOrEqual(
1831                 index, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
1832                 
1833             if (!m_node->arrayMode().isOutOfBounds())
1834                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
1835             else {
1836                 LBasicBlock outOfBoundsCase =
1837                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
1838                 LBasicBlock holeCase =
1839                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
1840                     
1841                 m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase);
1842                     
1843                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
1844                     
1845                 vmCall(
1846                     m_out.operation(slowPathFunction),
1847                     m_callFrame, base, index, value);
1848                     
1849                 m_out.jump(continuation);
1850                     
1851                 m_out.appendTo(holeCase, innerLastNext);
1852             }
1853             
1854             m_out.store32(
1855                 m_out.add(index, m_out.int32One),
1856                 storage, m_heaps.Butterfly_publicLength);
1857                 
1858             m_out.jump(performStore);
1859             m_out.appendTo(performStore, lastNext);
1860         }
1861     }
1862     
1863     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
1864     {
1865         Vector<SwitchCase> cases;
1866         for (unsigned i = 0; i < data->cases.size(); ++i) {
1867             cases.append(SwitchCase(
1868                 constInt(type, data->cases[i].value.switchLookupValue()),
1869                 lowBlock(data->cases[i].target)));
1870         }
1871         
1872         m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough));
1873     }
1874     
1875     void speculateBackward(
1876         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1877     {
1878         appendOSRExit(
1879             kind, lowValue, highValue, failCondition, BackwardSpeculation, FormattedValue());
1880     }
1881     
1882     void speculateForward(
1883         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
1884         const FormattedValue& recovery)
1885     {
1886         appendOSRExit(
1887             kind, lowValue, highValue, failCondition, ForwardSpeculation, recovery);
1888     }
1889     
1890     void speculate(
1891         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
1892     {
1893         appendOSRExit(
1894             kind, lowValue, highValue, failCondition, m_direction, FormattedValue());
1895     }
1896     
1897     void terminate(ExitKind kind)
1898     {
1899         speculate(kind, noValue(), 0, m_out.booleanTrue);
1900     }
1901     
1902     void backwardTypeCheck(
1903         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1904         LValue failCondition)
1905     {
1906         appendTypeCheck(
1907             lowValue, highValue, typesPassedThrough, failCondition, BackwardSpeculation,
1908             FormattedValue());
1909     }
1910     
1911     void forwardTypeCheck(
1912         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1913         LValue failCondition)
1914     {
1915         appendTypeCheck(
1916             lowValue, highValue, typesPassedThrough, failCondition, ForwardSpeculation,
1917             FormattedValue());
1918     }
1919     
1920     void typeCheck(
1921         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1922         LValue failCondition)
1923     {
1924         appendTypeCheck(
1925             lowValue, highValue, typesPassedThrough, failCondition, m_direction,
1926             FormattedValue());
1927     }
1928     
1929     void appendTypeCheck(
1930         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
1931         LValue failCondition, SpeculationDirection direction, FormattedValue recovery)
1932     {
1933         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
1934             return;
1935         ASSERT(mayHaveTypeCheck(highValue.useKind()));
1936         appendOSRExit(BadType, lowValue, highValue.node(), failCondition, direction, recovery);
1937         m_interpreter.filter(highValue, typesPassedThrough);
1938     }
1939     
1940     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1941     {
1942         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
1943         
1944         LoweredNodeValue value = m_int32Values.get(edge.node());
1945         if (isValid(value))
1946             return value.value();
1947         
1948         value = m_jsValueValues.get(edge.node());
1949         if (isValid(value)) {
1950             LValue boxedResult = value.value();
1951             FTL_TYPE_CHECK(
1952                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
1953             LValue result = unboxInt32(boxedResult);
1954             setInt32(edge.node(), result);
1955             return result;
1956         }
1957
1958         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
1959         terminate(Uncountable);
1960         return m_out.int32Zero;
1961     }
1962     
1963     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1964     {
1965         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
1966         
1967         LoweredNodeValue value = m_jsValueValues.get(edge.node());
1968         if (isValid(value)) {
1969             LValue uncheckedValue = value.value();
1970             FTL_TYPE_CHECK(
1971                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
1972             return uncheckedValue;
1973         }
1974         
1975         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
1976         terminate(Uncountable);
1977         return m_out.intPtrZero;
1978     }
1979     
1980     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1981     {
1982         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
1983         
1984         LValue result = lowCell(edge, mode);
1985         speculateObject(edge, result);
1986         return result;
1987     }
1988     
1989     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1990     {
1991         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse);
1992         
1993         LValue result = lowCell(edge, mode);
1994         speculateString(edge, result);
1995         return result;
1996     }
1997     
1998     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
1999     {
2000         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2001         
2002         LValue result = lowCell(edge, mode);
2003         speculateNonNullObject(edge, result);
2004         return result;
2005     }
2006     
2007     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2008     {
2009         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
2010         
2011         LoweredNodeValue value = m_booleanValues.get(edge.node());
2012         if (isValid(value))
2013             return value.value();
2014         
2015         value = m_jsValueValues.get(edge.node());
2016         if (isValid(value)) {
2017             LValue unboxedResult = value.value();
2018             FTL_TYPE_CHECK(
2019                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
2020             LValue result = unboxBoolean(unboxedResult);
2021             setBoolean(edge.node(), result);
2022             return result;
2023         }
2024         
2025         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
2026         terminate(Uncountable);
2027         return m_out.booleanFalse;
2028     }
2029     
2030     LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2031     {
2032         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind()));
2033         
2034         LoweredNodeValue value = m_doubleValues.get(edge.node());
2035         if (isValid(value))
2036             return value.value();
2037         
2038         value = m_int32Values.get(edge.node());
2039         if (isValid(value)) {
2040             LValue result = m_out.intToDouble(value.value());
2041             setDouble(edge.node(), result);
2042             return result;
2043         }
2044         
2045         value = m_jsValueValues.get(edge.node());
2046         if (isValid(value)) {
2047             LValue boxedResult = value.value();
2048             
2049             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case"));
2050             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case"));
2051             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation"));
2052             
2053             m_out.branch(isNotInt32(boxedResult), doubleCase, intCase);
2054             
2055             LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
2056             
2057             ValueFromBlock intToDouble = m_out.anchor(
2058                 m_out.intToDouble(unboxInt32(boxedResult)));
2059             m_out.jump(continuation);
2060             
2061             m_out.appendTo(doubleCase, continuation);
2062             
2063             FTL_TYPE_CHECK(
2064                 jsValueValue(boxedResult), edge, SpecNumber, isCellOrMisc(boxedResult));
2065             
2066             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult));
2067             m_out.jump(continuation);
2068             
2069             m_out.appendTo(continuation, lastNext);
2070             
2071             LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
2072             
2073             setDouble(edge.node(), result);
2074             return result;
2075         }
2076         
2077         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecNumber));
2078         terminate(Uncountable);
2079         return m_out.doubleZero;
2080     }
2081     
2082     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2083     {
2084         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2085         
2086         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2087         if (isValid(value))
2088             return value.value();
2089         
2090         value = m_int32Values.get(edge.node());
2091         if (isValid(value)) {
2092             LValue result = boxInt32(value.value());
2093             setJSValue(edge.node(), result);
2094             return result;
2095         }
2096         
2097         value = m_booleanValues.get(edge.node());
2098         if (isValid(value)) {
2099             LValue result = boxBoolean(value.value());
2100             setJSValue(edge.node(), result);
2101             return result;
2102         }
2103         
2104         value = m_doubleValues.get(edge.node());
2105         if (isValid(value)) {
2106             LValue result = boxDouble(value.value());
2107             setJSValue(edge.node(), result);
2108             return result;
2109         }
2110         
2111         RELEASE_ASSERT_NOT_REACHED();
2112         return 0;
2113     }
2114     
2115     LValue lowStorage(Edge edge)
2116     {
2117         LoweredNodeValue value = m_storageValues.get(edge.node());
2118         if (isValid(value))
2119             return value.value();
2120         
2121         LValue result = lowCell(edge);
2122         setStorage(edge.node(), result);
2123         return result;
2124     }
2125     
2126     LValue isNotInt32(LValue jsValue)
2127     {
2128         return m_out.below(jsValue, m_tagTypeNumber);
2129     }
2130     LValue unboxInt32(LValue jsValue)
2131     {
2132         return m_out.castToInt32(jsValue);
2133     }
2134     LValue boxInt32(LValue value)
2135     {
2136         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
2137     }
2138     
2139     LValue isCellOrMisc(LValue jsValue)
2140     {
2141         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
2142     }
2143     LValue unboxDouble(LValue jsValue)
2144     {
2145         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
2146     }
2147     LValue boxDouble(LValue doubleValue)
2148     {
2149         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
2150     }
2151     
2152     LValue isNotCell(LValue jsValue)
2153     {
2154         return m_out.testNonZero64(jsValue, m_tagMask);
2155     }
2156     
2157     LValue isNotBoolean(LValue jsValue)
2158     {
2159         return m_out.testNonZero64(
2160             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
2161             m_out.constInt64(~1));
2162     }
2163     LValue unboxBoolean(LValue jsValue)
2164     {
2165         // We want to use a cast that guarantees that LLVM knows that even the integer
2166         // value is just 0 or 1. But for now we do it the dumb way.
2167         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
2168     }
2169     LValue boxBoolean(LValue value)
2170     {
2171         return m_out.select(
2172             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
2173     }
2174
2175     void speculate(Edge edge)
2176     {
2177         switch (edge.useKind()) {
2178         case UntypedUse:
2179             break;
2180         case KnownInt32Use:
2181         case KnownNumberUse:
2182             ASSERT(!m_interpreter.needsTypeCheck(edge));
2183             break;
2184         case Int32Use:
2185             speculateInt32(edge);
2186             break;
2187         case CellUse:
2188             speculateCell(edge);
2189             break;
2190         case KnownCellUse:
2191             ASSERT(!m_interpreter.needsTypeCheck(edge));
2192             break;
2193         case ObjectUse:
2194             speculateObject(edge);
2195             break;
2196         case StringUse:
2197             speculateString(edge);
2198             break;
2199         case RealNumberUse:
2200             speculateRealNumber(edge);
2201             break;
2202         case NumberUse:
2203             speculateNumber(edge);
2204             break;
2205         case BooleanUse:
2206             speculateBoolean(edge);
2207             break;
2208         default:
2209             RELEASE_ASSERT_NOT_REACHED();
2210         }
2211     }
2212     
2213     void speculate(Node*, Edge edge)
2214     {
2215         speculate(edge);
2216     }
2217     
2218     void speculateInt32(Edge edge)
2219     {
2220         lowInt32(edge);
2221     }
2222     
2223     void speculateCell(Edge edge)
2224     {
2225         lowCell(edge);
2226     }
2227     
2228     LValue isObject(LValue cell)
2229     {
2230         return m_out.notEqual(
2231             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2232             m_out.constIntPtr(vm().stringStructure.get()));
2233     }
2234     
2235     LValue isNotString(LValue cell)
2236     {
2237         return isObject(cell);
2238     }
2239     
2240     LValue isString(LValue cell)
2241     {
2242         return m_out.equal(
2243             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2244             m_out.constIntPtr(vm().stringStructure.get()));
2245     }
2246     
2247     LValue isNotObject(LValue cell)
2248     {
2249         return isString(cell);
2250     }
2251     
2252     void speculateObject(Edge edge, LValue cell)
2253     {
2254         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
2255     }
2256     
2257     void speculateObject(Edge edge)
2258     {
2259         speculateObject(edge, lowCell(edge));
2260     }
2261     
2262     void speculateString(Edge edge, LValue cell)
2263     {
2264         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
2265     }
2266     
2267     void speculateString(Edge edge)
2268     {
2269         speculateString(edge, lowCell(edge));
2270     }
2271     
2272     void speculateNonNullObject(Edge edge, LValue cell)
2273     {
2274         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
2275         FTL_TYPE_CHECK(
2276             jsValueValue(cell), edge, SpecObject, 
2277             m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get())));
2278         if (masqueradesAsUndefinedWatchpointIfIsStillValid())
2279             return;
2280         
2281         speculate(
2282             BadType, jsValueValue(cell), edge.node(),
2283             m_out.testNonZero8(
2284                 m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2285                 m_out.constInt8(MasqueradesAsUndefined)));
2286     }
2287     
2288     void speculateNumber(Edge edge)
2289     {
2290         // Do an early return here because lowDouble() can create a lot of control flow.
2291         if (!m_interpreter.needsTypeCheck(edge))
2292             return;
2293         
2294         lowDouble(edge);
2295     }
2296     
2297     void speculateRealNumber(Edge edge)
2298     {
2299         // Do an early return here because lowDouble() can create a lot of control flow.
2300         if (!m_interpreter.needsTypeCheck(edge))
2301             return;
2302         
2303         LValue value = lowDouble(edge);
2304         FTL_TYPE_CHECK(
2305             doubleValue(value), edge, SpecRealNumber,
2306             m_out.doubleNotEqualOrUnordered(value, value));
2307     }
2308     
2309     void speculateBoolean(Edge edge)
2310     {
2311         lowBoolean(edge);
2312     }
2313
2314     bool masqueradesAsUndefinedWatchpointIsStillValid()
2315     {
2316         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin);
2317     }
2318     
2319     bool masqueradesAsUndefinedWatchpointIfIsStillValid()
2320     {
2321         if (!masqueradesAsUndefinedWatchpointIsStillValid())
2322             return false;
2323         
2324         // FIXME: Implement masquerades-as-undefined watchpoints.
2325         // https://bugs.webkit.org/show_bug.cgi?id=113647
2326         return true;
2327     }
2328     
2329     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
2330     
2331     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
2332     {
2333         callPreflight();
2334         LValue result = m_out.call(function);
2335         callCheck(mode);
2336         return result;
2337     }
2338     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
2339     {
2340         callPreflight();
2341         LValue result = m_out.call(function, arg1);
2342         callCheck(mode);
2343         return result;
2344     }
2345     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
2346     {
2347         callPreflight();
2348         LValue result = m_out.call(function, arg1, arg2);
2349         callCheck(mode);
2350         return result;
2351     }
2352     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
2353     {
2354         callPreflight();
2355         LValue result = m_out.call(function, arg1, arg2, arg3);
2356         callCheck(mode);
2357         return result;
2358     }
2359     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
2360     {
2361         callPreflight();
2362         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
2363         callCheck(mode);
2364         return result;
2365     }
2366     
2367     void callPreflight(CodeOrigin codeOrigin)
2368     {
2369         m_out.store32(
2370             m_out.constInt32(
2371                 CallFrame::Location::encodeAsCodeOriginIndex(
2372                     codeBlock()->addCodeOrigin(codeOrigin))), 
2373             tagFor(JSStack::ArgumentCount));
2374     }
2375     void callPreflight()
2376     {
2377         callPreflight(m_node->codeOrigin);
2378     }
2379     
2380     void callCheck(ExceptionCheckMode mode = CheckExceptions)
2381     {
2382         if (mode == NoExceptions)
2383             return;
2384         
2385         LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception"));
2386         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
2387         
2388         m_out.branch(
2389             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
2390             didHaveException, continuation);
2391         
2392         LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation);
2393         // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622
2394         m_out.crash();
2395         
2396         m_out.appendTo(continuation, lastNext);
2397     }
2398     
2399     bool isLive(Node* node)
2400     {
2401         return m_live.contains(node);
2402     }
2403     
2404     void use(Edge edge)
2405     {
2406         ASSERT(edge->hasResult());
2407         if (!edge.doesKill())
2408             return;
2409         m_live.remove(edge.node());
2410     }
2411     
2412     // Wrapper used only for DFG_NODE_DO_TO_CHILDREN
2413     void use(Node*, Edge edge)
2414     {
2415         use(edge);
2416     }
2417     
2418     LBasicBlock lowBlock(BasicBlock* block)
2419     {
2420         return m_blocks.get(block);
2421     }
2422     
2423     void initializeOSRExitStateForBlock()
2424     {
2425         for (unsigned i = m_valueSources.size(); i--;) {
2426             FlushFormat format = m_highBlock->ssa->flushFormatAtHead[i];
2427             switch (format) {
2428             case DeadFlush: {
2429                 // Must consider available nodes instead.
2430                 Node* node = m_highBlock->ssa->availabilityAtHead[i];
2431                 if (!node) {
2432                     m_valueSources[i] = ValueSource(SourceIsDead);
2433                     break;
2434                 }
2435                 
2436                 m_valueSources[i] = ValueSource(node);
2437                 break;
2438             }
2439                 
2440             case FlushedInt32:
2441                 m_valueSources[i] = ValueSource(Int32InJSStack);
2442                 break;
2443                 
2444             case FlushedDouble:
2445                 m_valueSources[i] = ValueSource(DoubleInJSStack);
2446                 break;
2447                 
2448             case FlushedCell:
2449             case FlushedBoolean:
2450             case FlushedJSValue:
2451                 m_valueSources[i] = ValueSource(ValueInJSStack);
2452                 break;
2453             }
2454         }
2455     }
2456     
2457     void appendOSRExit(
2458         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2459         SpeculationDirection direction, FormattedValue recovery)
2460     {
2461         if (Options::ftlTrapsOnOSRExit()) {
2462             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2463             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2464             
2465             m_out.branch(failCondition, failCase, continuation);
2466             
2467             LBasicBlock lastNext = m_out.appendTo(failCase, continuation);
2468             m_out.trap();
2469             m_out.unreachable();
2470             
2471             m_out.appendTo(continuation, lastNext);
2472             return;
2473         }
2474         
2475         if (verboseCompilationEnabled())
2476             dataLog("    OSR exit with value sources: ", m_valueSources, "\n");
2477         
2478         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.osrExit.size());
2479         unsigned index = m_ftlState.osrExit.size();
2480         
2481         m_ftlState.jitCode->osrExit.append(OSRExit(
2482             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
2483             m_codeOriginForExitTarget, m_codeOriginForExitProfile, m_lastSetOperand,
2484             m_valueSources.numberOfArguments(), m_valueSources.numberOfLocals()));
2485         m_ftlState.osrExit.append(OSRExitCompilationInfo());
2486         
2487         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
2488         OSRExitCompilationInfo& info = m_ftlState.osrExit.last();
2489
2490         LBasicBlock lastNext = 0;
2491         LBasicBlock continuation = 0;
2492         
2493         if (!Options::useLLVMOSRExitIntrinsic()) {
2494             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2495             continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2496             
2497             m_out.branch(failCondition, failCase, continuation);
2498
2499             m_out.appendTo(m_prologue);
2500             info.m_thunkAddress = buildAlloca(m_out.m_builder, m_out.intPtr);
2501         
2502             lastNext = m_out.appendTo(failCase, continuation);
2503         }
2504         
2505         if (Options::ftlOSRExitOmitsMarshalling()) {
2506             m_out.call(
2507                 m_out.intToPtr(
2508                     m_out.get(info.m_thunkAddress),
2509                     pointerType(functionType(m_out.voidType))));
2510         } else
2511             emitOSRExitCall(failCondition, index, exit, info, lowValue, direction, recovery);
2512         
2513         if (!Options::useLLVMOSRExitIntrinsic()) {
2514             m_out.unreachable();
2515             
2516             m_out.appendTo(continuation, lastNext);
2517         
2518             m_exitThunkGenerator.emitThunk(index);
2519         }
2520     }
2521     
2522     void emitOSRExitCall(
2523         LValue failCondition, unsigned index, OSRExit& exit, OSRExitCompilationInfo& info,
2524         FormattedValue lowValue, SpeculationDirection direction, FormattedValue recovery)
2525     {
2526         ExitArgumentList arguments;
2527         
2528         if (Options::useLLVMOSRExitIntrinsic()) {
2529             arguments.append(failCondition);
2530             arguments.append(m_out.constInt32(index));
2531         }
2532         
2533         arguments.append(m_callFrame);
2534         if (!!lowValue)
2535             arguments.append(lowValue.value());
2536         
2537         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
2538             ValueSource source = m_valueSources[i];
2539             
2540             switch (source.kind()) {
2541             case ValueInJSStack:
2542                 exit.m_values[i] = ExitValue::inJSStack();
2543                 break;
2544             case Int32InJSStack:
2545                 exit.m_values[i] = ExitValue::inJSStackAsInt32();
2546                 break;
2547             case DoubleInJSStack:
2548                 exit.m_values[i] = ExitValue::inJSStackAsDouble();
2549                 break;
2550             case SourceIsDead:
2551                 exit.m_values[i] = ExitValue::dead();
2552                 break;
2553             case HaveNode:
2554                 addExitArgumentForNode(exit, arguments, i, source.node());
2555                 break;
2556             default:
2557                 RELEASE_ASSERT_NOT_REACHED();
2558                 break;
2559             }
2560         }
2561         
2562         if (verboseCompilationEnabled())
2563             dataLog("        Exit values: ", exit.m_values, "\n");
2564         
2565         if (direction == ForwardSpeculation) {
2566             ASSERT(m_node);
2567             exit.convertToForward(m_highBlock, m_node, m_nodeIndex, recovery, arguments);
2568         }
2569         
2570         // So, the really lame thing here is that we have to build an LLVM function type.
2571         // Booo.
2572         Vector<LType, 16> argumentTypes;
2573         for (unsigned i = 0; i < arguments.size(); ++i)
2574             argumentTypes.append(typeOf(arguments[i]));
2575         
2576         if (Options::useLLVMOSRExitIntrinsic()) {
2577             m_out.call(m_out.osrExitIntrinsic(), arguments);
2578             return;
2579         }
2580         
2581         m_out.call(
2582             m_out.intToPtr(
2583                 m_out.get(info.m_thunkAddress),
2584                 pointerType(functionType(m_out.voidType, argumentTypes))),
2585             arguments);
2586     }
2587     
2588     void addExitArgumentForNode(
2589         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
2590     {
2591         ASSERT(node->shouldGenerate());
2592         ASSERT(node->hasResult());
2593
2594         if (tryToSetConstantExitArgument(exit, index, node))
2595             return;
2596         
2597         if (!isLive(node)) {
2598             bool found = false;
2599             
2600             if (needsOSRBackwardRewiring(node->op())) {
2601                 node = node->child1().node();
2602                 if (tryToSetConstantExitArgument(exit, index, node))
2603                     return;
2604                 if (isLive(node))
2605                     found = true;
2606             }
2607             
2608             if (!found) {
2609                 Node* int32ToDouble = 0;
2610                 Node* valueToInt32 = 0;
2611                 Node* uint32ToNumber = 0;
2612                 Node* doubleAsInt32 = 0;
2613                 
2614                 HashSet<Node*>::iterator iter = m_live.begin();
2615                 HashSet<Node*>::iterator end = m_live.end();
2616                 for (; iter != end; ++iter) {
2617                     Node* candidate = *iter;
2618                     if (!candidate->child1())
2619                         continue;
2620                     if (candidate->child1() != node)
2621                         continue;
2622                     switch (candidate->op()) {
2623                     case Int32ToDouble:
2624                         int32ToDouble = candidate;
2625                         break;
2626                     case ValueToInt32:
2627                         valueToInt32 = candidate;
2628                         break;
2629                     case UInt32ToNumber:
2630                         uint32ToNumber = candidate;
2631                         break;
2632                     case DoubleAsInt32:
2633                         uint32ToNumber = candidate;
2634                         break;
2635                     default:
2636                         ASSERT(!needsOSRForwardRewiring(candidate->op()));
2637                         break;
2638                     }
2639                 }
2640                 
2641                 if (doubleAsInt32)
2642                     node = doubleAsInt32;
2643                 else if (int32ToDouble)
2644                     node = int32ToDouble;
2645                 else if (valueToInt32)
2646                     node = valueToInt32;
2647                 else if (uint32ToNumber)
2648                     node = uint32ToNumber;
2649                 
2650                 if (isLive(node))
2651                     found = true;
2652             }
2653             
2654             if (!found) {
2655                 exit.m_values[index] = ExitValue::dead();
2656                 return;
2657             }
2658         }
2659         
2660         ASSERT(isLive(node));
2661
2662         LoweredNodeValue value = m_int32Values.get(node);
2663         if (isValid(value)) {
2664             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
2665             return;
2666         }
2667         
2668         value = m_booleanValues.get(node);
2669         if (isValid(value)) {
2670             addExitArgument(exit, arguments, index, ValueFormatBoolean, value.value());
2671             return;
2672         }
2673         
2674         value = m_jsValueValues.get(node);
2675         if (isValid(value)) {
2676             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
2677             return;
2678         }
2679         
2680         value = m_doubleValues.get(node);
2681         if (isValid(value)) {
2682             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
2683             return;
2684         }
2685
2686         dataLog("Cannot find value for node: ", node, "\n");
2687         RELEASE_ASSERT_NOT_REACHED();
2688     }
2689     
2690     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
2691     {
2692         if (!node)
2693             return false;
2694         
2695         switch (node->op()) {
2696         case JSConstant:
2697         case WeakJSConstant:
2698             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
2699             return true;
2700         case PhantomArguments:
2701             // FIXME: implement PhantomArguments.
2702             // https://bugs.webkit.org/show_bug.cgi?id=113986
2703             RELEASE_ASSERT_NOT_REACHED();
2704             return true;
2705         default:
2706             return false;
2707         }
2708     }
2709     
2710     void addExitArgument(
2711         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
2712         LValue value)
2713     {
2714         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
2715         arguments.append(value);
2716     }
2717     
2718     void linkOSRExitsAndCompleteInitializationBlocks()
2719     {
2720         MacroAssemblerCodeRef osrExitThunk =
2721             vm().getCTIStub(osrExitGenerationThunkGenerator);
2722         CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
2723         
2724         m_out.appendTo(m_prologue);
2725         m_out.jump(m_initialization);
2726         
2727         m_out.appendTo(m_initialization);
2728         
2729         if (m_exitThunkGenerator.didThings()) {
2730             OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
2731                 vm(), &m_exitThunkGenerator, m_ftlState.graph.m_codeBlock,
2732                 JITCompilationMustSucceed));
2733         
2734             ASSERT(m_ftlState.osrExit.size() == m_ftlState.jitCode->osrExit.size());
2735         
2736             for (unsigned i = 0; i < m_ftlState.osrExit.size(); ++i) {
2737                 OSRExitCompilationInfo& info = m_ftlState.osrExit[i];
2738                 OSRExit& exit = m_ftlState.jitCode->osrExit[i];
2739             
2740                 linkBuffer->link(info.m_thunkJump, target);
2741             
2742                 m_out.set(
2743                     m_out.constIntPtr(
2744                         linkBuffer->locationOf(info.m_thunkLabel).executableAddress()),
2745                     info.m_thunkAddress);
2746             
2747                 exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
2748             }
2749         
2750             m_ftlState.finalizer->initializeExitThunksLinkBuffer(linkBuffer.release());
2751         }
2752
2753         m_out.jump(lowBlock(m_graph.block(0)));
2754     }
2755     
2756     void observeMovHint(Node* node)
2757     {
2758         ASSERT(node->containsMovHint());
2759         ASSERT(node->op() != ZombieHint);
2760         
2761         int operand = node->local();
2762         
2763         m_lastSetOperand = operand;
2764         m_valueSources.operand(operand) = ValueSource(node->child1().node());
2765     }
2766     
2767     void setInt32(Node* node, LValue value)
2768     {
2769         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
2770     }
2771     void setJSValue(Node* node, LValue value)
2772     {
2773         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
2774     }
2775     void setBoolean(Node* node, LValue value)
2776     {
2777         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
2778     }
2779     void setStorage(Node* node, LValue value)
2780     {
2781         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
2782     }
2783     void setDouble(Node* node, LValue value)
2784     {
2785         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
2786     }
2787
2788     void setInt32(LValue value)
2789     {
2790         setInt32(m_node, value);
2791     }
2792     void setJSValue(LValue value)
2793     {
2794         setJSValue(m_node, value);
2795     }
2796     void setBoolean(LValue value)
2797     {
2798         setBoolean(m_node, value);
2799     }
2800     void setStorage(LValue value)
2801     {
2802         setStorage(m_node, value);
2803     }
2804     void setDouble(LValue value)
2805     {
2806         setDouble(m_node, value);
2807     }
2808     
2809     bool isValid(const LoweredNodeValue& value)
2810     {
2811         if (!value)
2812             return false;
2813         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
2814             return false;
2815         return true;
2816     }
2817     
2818     void addWeakReference(JSCell* target)
2819     {
2820         m_ftlState.jitCode->common.weakReferences.append(
2821             WriteBarrier<JSCell>(vm(), codeBlock()->ownerExecutable(), target));
2822     }
2823     
2824     LValue weakPointer(JSCell* pointer)
2825     {
2826         addWeakReference(pointer);
2827         return m_out.constIntPtr(pointer);
2828     }
2829     
2830     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
2831     {
2832         return m_out.address(base, m_heaps.variables[operand], offset);
2833     }
2834     TypedPointer payloadFor(LValue base, int operand)
2835     {
2836         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
2837     }
2838     TypedPointer tagFor(LValue base, int operand)
2839     {
2840         return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
2841     }
2842     TypedPointer addressFor(int operand)
2843     {
2844         return addressFor(m_callFrame, operand);
2845     }
2846     TypedPointer payloadFor(int operand)
2847     {
2848         return payloadFor(m_callFrame, operand);
2849     }
2850     TypedPointer tagFor(int operand)
2851     {
2852         return tagFor(m_callFrame, operand);
2853     }
2854     
2855     VM& vm() { return m_graph.m_vm; }
2856     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
2857     
2858     Graph& m_graph;
2859     State& m_ftlState;
2860     AbstractHeapRepository m_heaps;
2861     Output m_out;
2862     
2863     LBasicBlock m_prologue;
2864     LBasicBlock m_initialization;
2865     HashMap<BasicBlock*, LBasicBlock> m_blocks;
2866     
2867     LValue m_callFrame;
2868     LValue m_tagTypeNumber;
2869     LValue m_tagMask;
2870     
2871     HashMap<Node*, LoweredNodeValue> m_int32Values;
2872     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
2873     HashMap<Node*, LoweredNodeValue> m_booleanValues;
2874     HashMap<Node*, LoweredNodeValue> m_storageValues;
2875     HashMap<Node*, LoweredNodeValue> m_doubleValues;
2876     HashSet<Node*> m_live;
2877     
2878     HashMap<Node*, LValue> m_phis;
2879     
2880     Operands<ValueSource> m_valueSources;
2881     int m_lastSetOperand;
2882     ExitThunkGenerator m_exitThunkGenerator;
2883     
2884     InPlaceAbstractState m_state;
2885     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
2886     BasicBlock* m_highBlock;
2887     BasicBlock* m_nextHighBlock;
2888     LBasicBlock m_nextLowBlock;
2889     
2890     CodeOrigin m_codeOriginForExitTarget;
2891     CodeOrigin m_codeOriginForExitProfile;
2892     unsigned m_nodeIndex;
2893     Node* m_node;
2894     SpeculationDirection m_direction;
2895 };
2896
2897 void lowerDFGToLLVM(State& state)
2898 {
2899     LowerDFGToLLVM lowering(state);
2900     lowering.lower();
2901 }
2902
2903 } } // namespace JSC::FTL
2904
2905 #endif // ENABLE(FTL_JIT)
2906