Unreviewed, rolling out r156019 and r156020.
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToLLVM.cpp
1 /*
2  * Copyright (C) 2013 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToLLVM.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "CodeBlockWithJITType.h"
32 #include "DFGAbstractInterpreterInlines.h"
33 #include "DFGInPlaceAbstractState.h"
34 #include "FTLAbstractHeapRepository.h"
35 #include "FTLExitThunkGenerator.h"
36 #include "FTLForOSREntryJITCode.h"
37 #include "FTLFormattedValue.h"
38 #include "FTLLoweredNodeValue.h"
39 #include "FTLOutput.h"
40 #include "FTLThunks.h"
41 #include "FTLValueSource.h"
42 #include "LinkBuffer.h"
43 #include "OperandsInlines.h"
44 #include "Operations.h"
45 #include <wtf/ProcessID.h>
46
47 namespace JSC { namespace FTL {
48
49 using namespace DFG;
50
51 static int compileCounter;
52
53 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
54 // significantly less dead code.
55 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
56         FormattedValue _ftc_lowValue = (lowValue);                      \
57         Edge _ftc_highValue = (highValue);                              \
58         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
59         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
60             break;                                                      \
61         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
62     } while (false)
63
64 class LowerDFGToLLVM {
65 public:
66     LowerDFGToLLVM(State& state)
67         : m_graph(state.graph)
68         , m_ftlState(state)
69         , m_heaps(state.context)
70         , m_out(state.context)
71         , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
72         , m_lastSetOperand(std::numeric_limits<int>::max())
73         , m_exitThunkGenerator(state)
74         , m_state(state.graph)
75         , m_interpreter(state.graph, m_state)
76     {
77     }
78     
79     void lower()
80     {
81         CString name;
82         if (verboseCompilationEnabled()) {
83             name = toCString(
84                 "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
85                 "_", codeBlock()->hash());
86         } else
87             name = "jsBody";
88         
89         m_graph.m_dominators.computeIfNecessary(m_graph);
90         
91         m_ftlState.module =
92             LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
93         
94         m_ftlState.function = addFunction(
95             m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
96         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
97         
98         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
99         
100         m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
101         m_out.appendTo(m_prologue);
102         createPhiVariables();
103         
104         m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
105
106         m_callFrame = m_out.param(0);
107         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
108         m_tagMask = m_out.constInt64(TagMask);
109         
110         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
111             m_highBlock = m_graph.block(blockIndex);
112             if (!m_highBlock)
113                 continue;
114             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
115         }
116         
117         Vector<BasicBlock*> depthFirst;
118         m_graph.getBlocksInDepthFirstOrder(depthFirst);
119         for (unsigned i = 0; i < depthFirst.size(); ++i)
120             compileBlock(depthFirst[i]);
121         
122         // And now complete the initialization block.
123         linkOSRExitsAndCompleteInitializationBlocks();
124
125         if (Options::dumpLLVMIR())
126             dumpModule(m_ftlState.module);
127         
128         if (verboseCompilationEnabled())
129             m_ftlState.dumpState("after lowering");
130         if (validationEnabled())
131             verifyModule(m_ftlState.module);
132     }
133
134 private:
135     
136     void createPhiVariables()
137     {
138         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
139             BasicBlock* block = m_graph.block(blockIndex);
140             if (!block)
141                 continue;
142             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
143                 Node* node = block->at(nodeIndex);
144                 if (node->op() != Phi)
145                     continue;
146                 LType type;
147                 switch (node->flags() & NodeResultMask) {
148                 case NodeResultNumber:
149                     type = m_out.doubleType;
150                     break;
151                 case NodeResultInt32:
152                     type = m_out.int32;
153                     break;
154                 case NodeResultBoolean:
155                     type = m_out.boolean;
156                     break;
157                 case NodeResultJS:
158                     type = m_out.int64;
159                     break;
160                 default:
161                     RELEASE_ASSERT_NOT_REACHED();
162                     break;
163                 }
164                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
165             }
166         }
167     }
168     
169     void compileBlock(BasicBlock* block)
170     {
171         if (!block)
172             return;
173         
174         if (verboseCompilationEnabled())
175             dataLog("Compiling block ", *block, "\n");
176         
177         m_highBlock = block;
178         
179         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
180         
181         m_nextHighBlock = 0;
182         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
183             m_nextHighBlock = m_graph.block(nextBlockIndex);
184             if (m_nextHighBlock)
185                 break;
186         }
187         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
188         
189         // All of this effort to find the next block gives us the ability to keep the
190         // generated IR in roughly program order. This ought not affect the performance
191         // of the generated code (since we expect LLVM to reorder things) but it will
192         // make IR dumps easier to read.
193         m_out.appendTo(lowBlock, m_nextLowBlock);
194         
195         if (Options::ftlCrashes())
196             m_out.crashNonTerminal();
197         
198         if (!m_highBlock->cfaHasVisited) {
199             m_out.crash();
200             return;
201         }
202         
203         initializeOSRExitStateForBlock();
204         
205         m_live = block->ssa->liveAtHead;
206         
207         m_state.reset();
208         m_state.beginBasicBlock(m_highBlock);
209         
210         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
211             if (!compileNode(m_nodeIndex))
212                 break;
213         }
214     }
215     
216     bool compileNode(unsigned nodeIndex)
217     {
218         if (!m_state.isValid()) {
219             m_out.unreachable();
220             return false;
221         }
222         
223         m_node = m_highBlock->at(nodeIndex);
224         m_codeOriginForExitProfile = m_node->codeOrigin;
225         m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
226         
227         if (verboseCompilationEnabled())
228             dataLog("Lowering ", m_node, "\n");
229         
230         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
231         
232         m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
233         
234         switch (m_node->op()) {
235         case Upsilon:
236             compileUpsilon();
237             break;
238         case Phi:
239             compilePhi();
240             break;
241         case JSConstant:
242             compileJSConstant();
243             break;
244         case WeakJSConstant:
245             compileWeakJSConstant();
246             break;
247         case GetArgument:
248             compileGetArgument();
249             break;
250         case ExtractOSREntryLocal:
251             compileExtractOSREntryLocal();
252             break;
253         case GetLocal:
254             compileGetLocal();
255             break;
256         case SetLocal:
257             compileSetLocal();
258             break;
259         case MovHint:
260             compileMovHint();
261             break;
262         case ZombieHint:
263             compileZombieHint();
264             break;
265         case MovHintAndCheck:
266             compileMovHintAndCheck();
267             break;
268         case Phantom:
269             compilePhantom();
270             break;
271         case Flush:
272         case PhantomLocal:
273         case SetArgument:
274         case LoopHint:
275             break;
276         case ArithAdd:
277         case ValueAdd:
278             compileAdd();
279             break;
280         case ArithSub:
281             compileArithSub();
282             break;
283         case ArithMul:
284             compileArithMul();
285             break;
286         case ArithDiv:
287             compileArithDiv();
288             break;
289         case ArithMod:
290             compileArithMod();
291             break;
292         case ArithMin:
293         case ArithMax:
294             compileArithMinOrMax();
295             break;
296         case ArithAbs:
297             compileArithAbs();
298             break;
299         case ArithNegate:
300             compileArithNegate();
301             break;
302         case BitAnd:
303             compileBitAnd();
304             break;
305         case BitOr:
306             compileBitOr();
307             break;
308         case BitXor:
309             compileBitXor();
310             break;
311         case BitRShift:
312             compileBitRShift();
313             break;
314         case BitLShift:
315             compileBitLShift();
316             break;
317         case BitURShift:
318             compileBitURShift();
319             break;
320         case UInt32ToNumber:
321             compileUInt32ToNumber();
322             break;
323         case Int32ToDouble:
324             compileInt32ToDouble();
325             break;
326         case CheckStructure:
327             compileCheckStructure();
328             break;
329         case StructureTransitionWatchpoint:
330             compileStructureTransitionWatchpoint();
331             break;
332         case ArrayifyToStructure:
333             compileArrayifyToStructure();
334             break;
335         case PutStructure:
336             compilePutStructure();
337             break;
338         case PhantomPutStructure:
339             compilePhantomPutStructure();
340             break;
341         case GetButterfly:
342             compileGetButterfly();
343             break;
344         case GetIndexedPropertyStorage:
345             compileGetIndexedPropertyStorage();
346             break;
347         case CheckArray:
348             compileCheckArray();
349             break;
350         case GetArrayLength:
351             compileGetArrayLength();
352             break;
353         case GetByVal:
354             compileGetByVal();
355             break;
356         case PutByVal:
357         case PutByValAlias:
358             compilePutByVal();
359             break;
360         case GetByOffset:
361             compileGetByOffset();
362             break;
363         case PutByOffset:
364             compilePutByOffset();
365             break;
366         case GetGlobalVar:
367             compileGetGlobalVar();
368             break;
369         case PutGlobalVar:
370             compilePutGlobalVar();
371             break;
372         case GlobalVarWatchpoint:
373             compileGlobalVarWatchpoint();
374             break;
375         case GetMyScope:
376             compileGetMyScope();
377             break;
378         case SkipScope:
379             compileSkipScope();
380             break;
381         case GetClosureRegisters:
382             compileGetClosureRegisters();
383             break;
384         case GetClosureVar:
385             compileGetClosureVar();
386             break;
387         case PutClosureVar:
388             compilePutClosureVar();
389             break;
390         case CompareEq:
391             compileCompareEq();
392             break;
393         case CompareEqConstant:
394             compileCompareEqConstant();
395             break;
396         case CompareStrictEq:
397             compileCompareStrictEq();
398             break;
399         case CompareStrictEqConstant:
400             compileCompareStrictEqConstant();
401             break;
402         case CompareLess:
403             compileCompareLess();
404             break;
405         case CompareLessEq:
406             compileCompareLessEq();
407             break;
408         case CompareGreater:
409             compileCompareGreater();
410             break;
411         case CompareGreaterEq:
412             compileCompareGreaterEq();
413             break;
414         case LogicalNot:
415             compileLogicalNot();
416             break;
417         case Call:
418         case Construct:
419             compileCallOrConstruct();
420             break;
421         case Jump:
422             compileJump();
423             break;
424         case Branch:
425             compileBranch();
426             break;
427         case Switch:
428             compileSwitch();
429             break;
430         case Return:
431             compileReturn();
432             break;
433         case ForceOSRExit:
434             compileForceOSRExit();
435             break;
436         default:
437             RELEASE_ASSERT_NOT_REACHED();
438             break;
439         }
440         
441         if (m_node->shouldGenerate())
442             DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
443         
444         if (m_node->adjustedRefCount())
445             m_live.add(m_node);
446         
447         if (shouldExecuteEffects)
448             m_interpreter.executeEffects(nodeIndex);
449         
450         return true;
451     }
452     
453     void compileUpsilon()
454     {
455         LValue destination = m_phis.get(m_node->phi());
456         
457         switch (m_node->child1().useKind()) {
458         case NumberUse:
459             m_out.set(lowDouble(m_node->child1()), destination);
460             break;
461         case Int32Use:
462             m_out.set(lowInt32(m_node->child1()), destination);
463             break;
464         case BooleanUse:
465             m_out.set(lowBoolean(m_node->child1()), destination);
466             break;
467         case CellUse:
468             m_out.set(lowCell(m_node->child1()), destination);
469             break;
470         case UntypedUse:
471             m_out.set(lowJSValue(m_node->child1()), destination);
472             break;
473         default:
474             RELEASE_ASSERT_NOT_REACHED();
475             break;
476         }
477     }
478     
479     void compilePhi()
480     {
481         LValue source = m_phis.get(m_node);
482         
483         switch (m_node->flags() & NodeResultMask) {
484         case NodeResultNumber:
485             setDouble(m_out.get(source));
486             break;
487         case NodeResultInt32:
488             setInt32(m_out.get(source));
489             break;
490         case NodeResultBoolean:
491             setBoolean(m_out.get(source));
492             break;
493         case NodeResultJS:
494             setJSValue(m_out.get(source));
495             break;
496         default:
497             RELEASE_ASSERT_NOT_REACHED();
498             break;
499         }
500     }
501     
502     void compileJSConstant()
503     {
504         JSValue value = m_graph.valueOfJSConstant(m_node);
505         if (value.isDouble())
506             setDouble(m_out.constDouble(value.asDouble()));
507         else
508             setJSValue(m_out.constInt64(JSValue::encode(value)));
509     }
510     
511     void compileWeakJSConstant()
512     {
513         setJSValue(weakPointer(m_node->weakConstant()));
514     }
515     
516     void compileGetArgument()
517     {
518         VariableAccessData* variable = m_node->variableAccessData();
519         int operand = variable->operand();
520
521         LValue jsValue = m_out.load64(addressFor(operand));
522
523         switch (useKindFor(variable->flushFormat())) {
524         case Int32Use:
525             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
526             setInt32(unboxInt32(jsValue));
527             break;
528         case CellUse:
529             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
530             setJSValue(jsValue);
531             break;
532         case BooleanUse:
533             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
534             setBoolean(unboxBoolean(jsValue));
535             break;
536         case UntypedUse:
537             setJSValue(jsValue);
538             break;
539         default:
540             RELEASE_ASSERT_NOT_REACHED();
541             break;
542         }
543     }
544     
545     void compileExtractOSREntryLocal()
546     {
547         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
548             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
549         setJSValue(m_out.load64(m_out.absolute(buffer + operandToLocal(m_node->unlinkedLocal()))));
550     }
551     
552     void compileGetLocal()
553     {
554         // GetLocals arise only for captured variables.
555         
556         VariableAccessData* variable = m_node->variableAccessData();
557         AbstractValue& value = m_state.variables().operand(variable->local());
558         
559         RELEASE_ASSERT(variable->isCaptured());
560         
561         if (isInt32Speculation(value.m_type))
562             setInt32(m_out.load32(payloadFor(variable->local())));
563         else
564             setJSValue(m_out.load64(addressFor(variable->local())));
565     }
566     
567     void compileSetLocal()
568     {
569         observeMovHint(m_node);
570         
571         VariableAccessData* variable = m_node->variableAccessData();
572         switch (variable->flushFormat()) {
573         case FlushedJSValue: {
574             LValue value = lowJSValue(m_node->child1());
575             m_out.store64(value, addressFor(variable->local()));
576             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
577             return;
578         }
579             
580         case FlushedDouble: {
581             LValue value = lowDouble(m_node->child1());
582             m_out.storeDouble(value, addressFor(variable->local()));
583             m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
584             return;
585         }
586             
587         case FlushedInt32: {
588             LValue value = lowInt32(m_node->child1());
589             m_out.store32(value, payloadFor(variable->local()));
590             m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
591             return;
592         }
593             
594         case FlushedCell: {
595             LValue value = lowCell(m_node->child1());
596             m_out.store64(value, addressFor(variable->local()));
597             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
598             return;
599         }
600             
601         case FlushedBoolean: {
602             speculateBoolean(m_node->child1());
603             m_out.store64(
604                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
605                 addressFor(variable->local()));
606             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
607             return;
608         }
609             
610         case DeadFlush:
611             RELEASE_ASSERT_NOT_REACHED();
612         }
613         
614         RELEASE_ASSERT_NOT_REACHED();
615     }
616     
617     void compileMovHint()
618     {
619         observeMovHint(m_node);
620     }
621     
622     void compileZombieHint()
623     {
624         VariableAccessData* data = m_node->variableAccessData();
625         m_lastSetOperand = data->local();
626         m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
627     }
628     
629     void compileMovHintAndCheck()
630     {
631         observeMovHint(m_node);
632         speculate(m_node->child1());
633     }
634     
635     void compilePhantom()
636     {
637         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
638     }
639     
640     void compileAdd()
641     {
642         switch (m_node->binaryUseKind()) {
643         case Int32Use: {
644             LValue left = lowInt32(m_node->child1());
645             LValue right = lowInt32(m_node->child2());
646             
647             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags())) {
648                 setInt32(m_out.add(left, right));
649                 break;
650             }
651             
652             LValue result = m_out.addWithOverflow32(left, right);
653             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
654             setInt32(m_out.extractValue(result, 0));
655             break;
656         }
657             
658         case NumberUse: {
659             setDouble(
660                 m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
661             break;
662         }
663             
664         default:
665             RELEASE_ASSERT_NOT_REACHED();
666             break;
667         }
668     }
669     
670     void compileArithSub()
671     {
672         switch (m_node->binaryUseKind()) {
673         case Int32Use: {
674             LValue left = lowInt32(m_node->child1());
675             LValue right = lowInt32(m_node->child2());
676             
677             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags())) {
678                 setInt32(m_out.sub(left, right));
679                 break;
680             }
681             
682             LValue result = m_out.subWithOverflow32(left, right);
683             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
684             setInt32(m_out.extractValue(result, 0));
685             break;
686         }
687             
688         case NumberUse: {
689             setDouble(
690                 m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
691             break;
692         }
693             
694         default:
695             RELEASE_ASSERT_NOT_REACHED();
696             break;
697         }
698     }
699     
700     void compileArithMul()
701     {
702         switch (m_node->binaryUseKind()) {
703         case Int32Use: {
704             LValue left = lowInt32(m_node->child1());
705             LValue right = lowInt32(m_node->child2());
706             
707             LValue result;
708             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags()))
709                 result = m_out.mul(left, right);
710             else {
711                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
712                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
713                 result = m_out.extractValue(overflowResult, 0);
714             }
715             
716             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
717                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
718                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
719                 
720                 m_out.branch(m_out.notZero32(result), continuation, slowCase);
721                 
722                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
723                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
724                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
725                 m_out.jump(continuation);
726                 m_out.appendTo(continuation, lastNext);
727             }
728             
729             setInt32(result);
730             break;
731         }
732             
733         case NumberUse: {
734             setDouble(
735                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
736             break;
737         }
738             
739         default:
740             RELEASE_ASSERT_NOT_REACHED();
741             break;
742         }
743     }
744     
745     void compileArithDiv()
746     {
747         switch (m_node->binaryUseKind()) {
748         case Int32Use: {
749             LValue numerator = lowInt32(m_node->child1());
750             LValue denominator = lowInt32(m_node->child2());
751             
752             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
753             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
754             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
755             
756             Vector<ValueFromBlock, 3> results;
757             
758             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
759             
760             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
761             
762             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
763             
764             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
765             
766             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
767                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
768                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
769                 m_out.jump(continuation);
770             } else {
771                 // This is the case where we convert the result to an int after we're done. So,
772                 // if the denominator is zero, then the result should be result should be zero.
773                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
774                 // check above) and the numerator is -2^31 then the result should be -2^31.
775                 
776                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
777                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
778                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
779                 
780                 m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
781                 
782                 m_out.appendTo(divByZero, notDivByZero);
783                 results.append(m_out.anchor(m_out.int32Zero));
784                 m_out.jump(done);
785                 
786                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
787                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
788                 
789                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
790                 results.append(m_out.anchor(neg2ToThe31));
791                 m_out.jump(done);
792             }
793             
794             m_out.appendTo(continuation, done);
795             
796             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
797                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
798                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
799                 
800                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
801                 
802                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
803                 
804                 speculate(
805                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
806                 
807                 m_out.jump(numeratorContinuation);
808                 
809                 m_out.appendTo(numeratorContinuation, innerLastNext);
810             }
811             
812             LValue divisionResult = m_out.div(numerator, denominator);
813             
814             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
815                 speculate(
816                     Overflow, noValue(), 0,
817                     m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
818             }
819             
820             results.append(m_out.anchor(divisionResult));
821             m_out.jump(done);
822             
823             m_out.appendTo(done, lastNext);
824             
825             setInt32(m_out.phi(m_out.int32, results));
826             break;
827         }
828             
829         case NumberUse: {
830             setDouble(
831                 m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
832             break;
833         }
834             
835         default:
836             RELEASE_ASSERT_NOT_REACHED();
837             break;
838         }
839     }
840     
841     void compileArithMod()
842     {
843         switch (m_node->binaryUseKind()) {
844         case Int32Use: {
845             LValue numerator = lowInt32(m_node->child1());
846             LValue denominator = lowInt32(m_node->child2());
847             
848             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
849             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
850             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
851             
852             Vector<ValueFromBlock, 3> results;
853             
854             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
855             
856             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
857             
858             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
859             
860             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
861             
862             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
863             // separate case for that. But it probably doesn't matter so much.
864             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
865                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
866                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
867                 m_out.jump(continuation);
868             } else {
869                 // This is the case where we convert the result to an int after we're done. So,
870                 // if the denominator is zero, then the result should be result should be zero.
871                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
872                 // check above) and the numerator is -2^31 then the result should be -2^31.
873                 
874                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
875                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
876                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
877                 
878                 m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
879                 
880                 m_out.appendTo(modByZero, notModByZero);
881                 results.append(m_out.anchor(m_out.int32Zero));
882                 m_out.jump(done);
883                 
884                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
885                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
886                 
887                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
888                 results.append(m_out.anchor(m_out.int32Zero));
889                 m_out.jump(done);
890             }
891             
892             m_out.appendTo(continuation, done);
893             
894             LValue remainder = m_out.rem(numerator, denominator);
895             
896             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
897                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
898                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
899                 
900                 m_out.branch(
901                     m_out.lessThan(numerator, m_out.int32Zero),
902                     negativeNumerator, numeratorContinuation);
903                 
904                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
905                 
906                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
907                 
908                 m_out.jump(numeratorContinuation);
909                 
910                 m_out.appendTo(numeratorContinuation, innerLastNext);
911             }
912             
913             results.append(m_out.anchor(remainder));
914             m_out.jump(done);
915             
916             m_out.appendTo(done, lastNext);
917             
918             setInt32(m_out.phi(m_out.int32, results));
919             break;
920         }
921             
922         case NumberUse: {
923             setDouble(
924                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
925             break;
926         }
927             
928         default:
929             RELEASE_ASSERT_NOT_REACHED();
930             break;
931         }
932     }
933
934     void compileArithMinOrMax()
935     {
936         switch (m_node->binaryUseKind()) {
937         case Int32Use: {
938             LValue left = lowInt32(m_node->child1());
939             LValue right = lowInt32(m_node->child2());
940             
941             setInt32(
942                 m_out.select(
943                     m_node->op() == ArithMin
944                         ? m_out.lessThan(left, right)
945                         : m_out.lessThan(right, left),
946                     left, right));
947             break;
948         }
949             
950         case NumberUse: {
951             LValue left = lowDouble(m_node->child1());
952             LValue right = lowDouble(m_node->child2());
953             
954             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
955             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
956             
957             Vector<ValueFromBlock, 2> results;
958             
959             results.append(m_out.anchor(left));
960             m_out.branch(
961                 m_node->op() == ArithMin
962                     ? m_out.doubleLessThan(left, right)
963                     : m_out.doubleGreaterThan(left, right),
964                 continuation, notLessThan);
965             
966             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
967             results.append(m_out.anchor(m_out.select(
968                 m_node->op() == ArithMin
969                     ? m_out.doubleGreaterThanOrEqual(left, right)
970                     : m_out.doubleLessThanOrEqual(left, right),
971                 right, m_out.constDouble(0.0 / 0.0))));
972             m_out.jump(continuation);
973             
974             m_out.appendTo(continuation, lastNext);
975             setDouble(m_out.phi(m_out.doubleType, results));
976             break;
977         }
978             
979         default:
980             RELEASE_ASSERT_NOT_REACHED();
981             break;
982         }
983     }
984     
985     void compileArithAbs()
986     {
987         switch (m_node->child1().useKind()) {
988         case Int32Use: {
989             LValue value = lowInt32(m_node->child1());
990             
991             LValue mask = m_out.aShr(value, m_out.constInt32(31));
992             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
993             
994             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
995             
996             setInt32(result);
997             break;
998         }
999             
1000         case NumberUse: {
1001             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1002             break;
1003         }
1004             
1005         default:
1006             RELEASE_ASSERT_NOT_REACHED();
1007             break;
1008         }
1009     }
1010     
1011     void compileArithNegate()
1012     {
1013         switch (m_node->child1().useKind()) {
1014         case Int32Use: {
1015             LValue value = lowInt32(m_node->child1());
1016             
1017             LValue result;
1018             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags()))
1019                 result = m_out.neg(value);
1020             else if (bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
1021                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1022                 // does the trick, though.
1023                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1024                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1025                 result = m_out.extractValue(overflowResult, 0);
1026             } else {
1027                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1028                 result = m_out.neg(value);
1029             }
1030             
1031             setInt32(result);
1032             break;
1033         }
1034             
1035         case NumberUse: {
1036             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1037             break;
1038         }
1039             
1040         default:
1041             RELEASE_ASSERT_NOT_REACHED();
1042             break;
1043         }
1044     }
1045     
1046     void compileBitAnd()
1047     {
1048         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1049     }
1050     
1051     void compileBitOr()
1052     {
1053         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1054     }
1055     
1056     void compileBitXor()
1057     {
1058         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1059     }
1060     
1061     void compileBitRShift()
1062     {
1063         setInt32(m_out.aShr(
1064             lowInt32(m_node->child1()),
1065             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1066     }
1067     
1068     void compileBitLShift()
1069     {
1070         setInt32(m_out.shl(
1071             lowInt32(m_node->child1()),
1072             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1073     }
1074     
1075     void compileBitURShift()
1076     {
1077         setInt32(m_out.lShr(
1078             lowInt32(m_node->child1()),
1079             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1080     }
1081     
1082     void compileUInt32ToNumber()
1083     {
1084         LValue value = lowInt32(m_node->child1());
1085
1086         if (!nodeCanSpeculateInt32(m_node->arithNodeFlags())) {
1087             setDouble(m_out.unsignedToDouble(value));
1088             return;
1089         }
1090         
1091         speculateForward(
1092             Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
1093             FormattedValue(ValueFormatUInt32, value));
1094         setInt32(value);
1095     }
1096     
1097     void compileInt32ToDouble()
1098     {
1099         // This node is tricky to compile in the DFG backend because it tries to
1100         // avoid converting child1 to a double in-place, as that would make subsequent
1101         // int uses of of child1 fail. But the FTL needs no such special magic, since
1102         // unlike the DFG backend, the FTL allows each node to have multiple
1103         // contemporaneous low-level representations. So, this gives child1 a double
1104         // representation and then forwards that representation to m_node.
1105         
1106         setDouble(lowDouble(m_node->child1()));
1107     }
1108     
1109     void compileCheckStructure()
1110     {
1111         LValue cell = lowCell(m_node->child1());
1112         
1113         ExitKind exitKind;
1114         if (m_node->child1()->op() == WeakJSConstant)
1115             exitKind = BadWeakConstantCache;
1116         else
1117             exitKind = BadCache;
1118         
1119         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1120         
1121         if (m_node->structureSet().size() == 1) {
1122             speculate(
1123                 exitKind, jsValueValue(cell), 0,
1124                 m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
1125             return;
1126         }
1127         
1128         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1129         
1130         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1131         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1132             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1133             m_out.branch(
1134                 m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
1135                 continuation, nextStructure);
1136             m_out.appendTo(nextStructure);
1137         }
1138         
1139         speculate(
1140             exitKind, jsValueValue(cell), 0,
1141             m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
1142         
1143         m_out.jump(continuation);
1144         m_out.appendTo(continuation, lastNext);
1145     }
1146     
1147     void compileStructureTransitionWatchpoint()
1148     {
1149         addWeakReference(m_node->structure());
1150         
1151         // FIXME: Implement structure transition watchpoints.
1152         // https://bugs.webkit.org/show_bug.cgi?id=113647
1153         
1154         speculateCell(m_node->child1());
1155     }
1156     
1157     void compileArrayifyToStructure()
1158     {
1159         LValue cell = lowCell(m_node->child1());
1160         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1161         
1162         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1163         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1164         
1165         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1166         
1167         m_out.branch(
1168             m_out.notEqual(structure, weakPointer(m_node->structure())),
1169             unexpectedStructure, continuation);
1170         
1171         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1172         
1173         if (property) {
1174             switch (m_node->arrayMode().type()) {
1175             case Array::Int32:
1176             case Array::Double:
1177             case Array::Contiguous:
1178                 speculate(
1179                     Uncountable, noValue(), 0,
1180                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1181                 break;
1182             default:
1183                 break;
1184             }
1185         }
1186         
1187         switch (m_node->arrayMode().type()) {
1188         case Array::Int32:
1189             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1190             break;
1191         case Array::Double:
1192             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1193             break;
1194         case Array::Contiguous:
1195             if (m_node->arrayMode().conversion() == Array::RageConvert)
1196                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1197             else
1198                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1199             break;
1200         case Array::ArrayStorage:
1201         case Array::SlowPutArrayStorage:
1202             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1203             break;
1204         default:
1205             RELEASE_ASSERT_NOT_REACHED();
1206             break;
1207         }
1208         
1209         structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1210         speculate(
1211             BadIndexingType, jsValueValue(cell), 0,
1212             m_out.notEqual(structure, weakPointer(m_node->structure())));
1213         m_out.jump(continuation);
1214         
1215         m_out.appendTo(continuation, lastNext);
1216     }
1217     
1218     void compilePutStructure()
1219     {
1220         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1221         
1222         m_out.store64(
1223             m_out.constIntPtr(m_node->structureTransitionData().newStructure),
1224             lowCell(m_node->child1()), m_heaps.JSCell_structure);
1225     }
1226     
1227     void compilePhantomPutStructure()
1228     {
1229         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1230     }
1231     
1232     void compileGetButterfly()
1233     {
1234         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1235     }
1236     
1237     void compileGetIndexedPropertyStorage()
1238     {
1239         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_vector));
1240     }
1241     
1242     void compileCheckArray()
1243     {
1244         Edge edge = m_node->child1();
1245         LValue cell = lowCell(edge);
1246         
1247         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge)))
1248             return;
1249         
1250         speculate(
1251             BadIndexingType, jsValueValue(cell), 0,
1252             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
1253     }
1254     
1255     void compileGetArrayLength()
1256     {
1257         switch (m_node->arrayMode().type()) {
1258         case Array::Int32:
1259         case Array::Double:
1260         case Array::Contiguous: {
1261             setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1262             return;
1263         }
1264             
1265         default:
1266             if (isTypedView(m_node->arrayMode().typedArrayType())) {
1267                 setInt32(
1268                     m_out.load32(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
1269                 return;
1270             }
1271             
1272             RELEASE_ASSERT_NOT_REACHED();
1273             return;
1274         }
1275     }
1276     
1277     void compileGetByVal()
1278     {
1279         LValue index = lowInt32(m_node->child2());
1280         LValue storage = lowStorage(m_node->child3());
1281         
1282         switch (m_node->arrayMode().type()) {
1283         case Array::Int32:
1284         case Array::Contiguous: {
1285             if (m_node->arrayMode().isInBounds()) {
1286                 speculate(
1287                     OutOfBounds, noValue(), 0,
1288                     m_out.aboveOrEqual(
1289                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1290                 
1291                 LValue result = m_out.load64(m_out.baseIndex(
1292                     m_node->arrayMode().type() == Array::Int32 ?
1293                         m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1294                     storage, m_out.zeroExt(index, m_out.intPtr),
1295                     m_state.forNode(m_node->child2()).m_value));
1296                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1297                 setJSValue(result);
1298                 return;
1299             }
1300             
1301             // FIXME: Implement hole/OOB loads in the FTL.
1302             // https://bugs.webkit.org/show_bug.cgi?id=118077
1303             RELEASE_ASSERT_NOT_REACHED();
1304             return;
1305         }
1306             
1307         case Array::Double: {
1308             if (m_node->arrayMode().isInBounds()) {
1309                 if (m_node->arrayMode().isSaneChain()) {
1310                     // FIXME: Implement structure transition watchpoints.
1311                     // https://bugs.webkit.org/show_bug.cgi?id=113647
1312                 }
1313             
1314                 speculate(
1315                     OutOfBounds, noValue(), 0,
1316                     m_out.aboveOrEqual(
1317                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1318                 
1319                 LValue result = m_out.loadDouble(m_out.baseIndex(
1320                     m_heaps.indexedDoubleProperties,
1321                     storage, m_out.zeroExt(index, m_out.intPtr),
1322                     m_state.forNode(m_node->child2()).m_value));
1323                 
1324                 if (!m_node->arrayMode().isSaneChain()) {
1325                     speculate(
1326                         LoadFromHole, noValue(), 0,
1327                         m_out.doubleNotEqualOrUnordered(result, result));
1328                 }
1329                 setDouble(result);
1330                 break;
1331             }
1332             
1333             // FIXME: Implement hole/OOB loads in the FTL.
1334             // https://bugs.webkit.org/show_bug.cgi?id=118077
1335             RELEASE_ASSERT_NOT_REACHED();
1336             return;
1337         }
1338             
1339         default: {
1340             TypedArrayType type = m_node->arrayMode().typedArrayType();
1341             
1342             if (isTypedView(type)) {
1343                 LValue array = lowCell(m_node->child1());
1344                 
1345                 speculate(
1346                     OutOfBounds, noValue(), 0,
1347                     m_out.aboveOrEqual(
1348                         index, m_out.load32(array, m_heaps.JSArrayBufferView_length)));
1349                 
1350                 TypedPointer pointer = TypedPointer(
1351                     m_heaps.typedArrayProperties,
1352                     m_out.add(
1353                         storage,
1354                         m_out.shl(
1355                             m_out.zeroExt(index, m_out.intPtr),
1356                             m_out.constIntPtr(logElementSize(type)))));
1357                 
1358                 if (isInt(type)) {
1359                     LValue result;
1360                     switch (elementSize(type)) {
1361                     case 1:
1362                         result = m_out.load8(pointer);
1363                         break;
1364                     case 2:
1365                         result = m_out.load16(pointer);
1366                         break;
1367                     case 4:
1368                         result = m_out.load32(pointer);
1369                         break;
1370                     default:
1371                         RELEASE_ASSERT_NOT_REACHED();
1372                     }
1373                     
1374                     if (elementSize(type) < 4) {
1375                         if (isSigned(type))
1376                             result = m_out.signExt(result, m_out.int32);
1377                         else
1378                             result = m_out.zeroExt(result, m_out.int32);
1379                         setInt32(result);
1380                         return;
1381                     }
1382                     
1383                     if (isSigned(type)) {
1384                         setInt32(result);
1385                         return;
1386                     }
1387                     
1388                     if (m_node->shouldSpeculateInt32()) {
1389                         speculateForward(
1390                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero),
1391                             uInt32Value(result));
1392                         setInt32(result);
1393                         return;
1394                     }
1395                     
1396                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
1397                     return;
1398                 }
1399             
1400                 ASSERT(isFloat(type));
1401                 
1402                 LValue result;
1403                 switch (type) {
1404                 case TypeFloat32:
1405                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
1406                     break;
1407                 case TypeFloat64:
1408                     result = m_out.loadDouble(pointer);
1409                     break;
1410                 default:
1411                     RELEASE_ASSERT_NOT_REACHED();
1412                 }
1413                 
1414                 result = m_out.select(
1415                     m_out.doubleEqual(result, result), result, m_out.constDouble(QNaN));
1416                 setDouble(result);
1417                 return;
1418             }
1419             
1420             RELEASE_ASSERT_NOT_REACHED();
1421             return;
1422         } }
1423     }
1424     
1425     void compilePutByVal()
1426     {
1427         Edge child1 = m_graph.varArgChild(m_node, 0);
1428         Edge child2 = m_graph.varArgChild(m_node, 1);
1429         Edge child3 = m_graph.varArgChild(m_node, 2);
1430         Edge child4 = m_graph.varArgChild(m_node, 3);
1431
1432         LValue base = lowCell(child1);
1433         LValue index = lowInt32(child2);
1434         LValue storage = lowStorage(child4);
1435         
1436         switch (m_node->arrayMode().type()) {
1437         case Array::Int32:
1438         case Array::Double:
1439         case Array::Contiguous: {
1440             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
1441             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
1442             
1443             switch (m_node->arrayMode().type()) {
1444             case Array::Int32:
1445             case Array::Contiguous: {
1446                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
1447                 
1448                 if (m_node->arrayMode().type() == Array::Int32)
1449                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
1450                 
1451                 TypedPointer elementPointer = m_out.baseIndex(
1452                     m_node->arrayMode().type() == Array::Int32 ?
1453                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1454                     storage, m_out.zeroExt(index, m_out.intPtr),
1455                     m_state.forNode(child2).m_value);
1456                 
1457                 if (m_node->op() == PutByValAlias) {
1458                     m_out.store64(value, elementPointer);
1459                     break;
1460                 }
1461                 
1462                 contiguousPutByValOutOfBounds(
1463                     codeBlock()->isStrictMode()
1464                     ? operationPutByValBeyondArrayBoundsStrict
1465                     : operationPutByValBeyondArrayBoundsNonStrict,
1466                     base, storage, index, value, continuation);
1467                 
1468                 m_out.store64(value, elementPointer);
1469                 break;
1470             }
1471                 
1472             case Array::Double: {
1473                 LValue value = lowDouble(child3);
1474                 
1475                 FTL_TYPE_CHECK(
1476                     doubleValue(value), child3, SpecRealNumber,
1477                     m_out.doubleNotEqualOrUnordered(value, value));
1478                 
1479                 TypedPointer elementPointer = m_out.baseIndex(
1480                     m_heaps.indexedDoubleProperties,
1481                     storage, m_out.zeroExt(index, m_out.intPtr),
1482                     m_state.forNode(child2).m_value);
1483                 
1484                 if (m_node->op() == PutByValAlias) {
1485                     m_out.storeDouble(value, elementPointer);
1486                     break;
1487                 }
1488                 
1489                 contiguousPutByValOutOfBounds(
1490                     codeBlock()->isStrictMode()
1491                     ? operationPutDoubleByValBeyondArrayBoundsStrict
1492                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1493                     base, storage, index, value, continuation);
1494                 
1495                 m_out.storeDouble(value, elementPointer);
1496                 break;
1497             }
1498                 
1499             default:
1500                 RELEASE_ASSERT_NOT_REACHED();
1501             }
1502
1503             m_out.jump(continuation);
1504             m_out.appendTo(continuation, outerLastNext);
1505             return;
1506         }
1507             
1508         default:
1509             TypedArrayType type = m_node->arrayMode().typedArrayType();
1510             
1511             if (isTypedView(type)) {
1512                 if (m_node->op() == PutByVal) {
1513                     speculate(
1514                         OutOfBounds, noValue(), 0,
1515                         m_out.aboveOrEqual(
1516                             index, m_out.load32(base, m_heaps.JSArrayBufferView_length)));
1517                 }
1518                 
1519                 TypedPointer pointer = TypedPointer(
1520                     m_heaps.typedArrayProperties,
1521                     m_out.add(
1522                         storage,
1523                         m_out.shl(
1524                             m_out.zeroExt(index, m_out.intPtr),
1525                             m_out.constIntPtr(logElementSize(type)))));
1526                 
1527                 if (isInt(type)) {
1528                     LValue intValue;
1529                     switch (child3.useKind()) {
1530                     case Int32Use: {
1531                         intValue = lowInt32(child3);
1532                         if (isClamped(type)) {
1533                             ASSERT(elementSize(type) == 1);
1534                             
1535                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
1536                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
1537                             
1538                             Vector<ValueFromBlock, 2> intValues;
1539                             intValues.append(m_out.anchor(m_out.int32Zero));
1540                             m_out.branch(
1541                                 m_out.lessThan(intValue, m_out.int32Zero),
1542                                 continuation, atLeastZero);
1543                             
1544                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
1545                             
1546                             intValues.append(m_out.anchor(m_out.select(
1547                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
1548                                 m_out.constInt32(255),
1549                                 intValue)));
1550                             m_out.jump(continuation);
1551                             
1552                             m_out.appendTo(continuation, lastNext);
1553                             intValue = m_out.phi(m_out.int32, intValues);
1554                         }
1555                         break;
1556                     }
1557                         
1558                     case NumberUse: {
1559                         LValue doubleValue = lowDouble(child3);
1560                         
1561                         if (isClamped(type)) {
1562                             ASSERT(elementSize(type) == 1);
1563                             
1564                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
1565                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
1566                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
1567                             
1568                             Vector<ValueFromBlock, 3> intValues;
1569                             intValues.append(m_out.anchor(m_out.int32Zero));
1570                             m_out.branch(
1571                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
1572                                 continuation, atLeastZero);
1573                             
1574                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
1575                             intValues.append(m_out.anchor(m_out.constInt32(255)));
1576                             m_out.branch(
1577                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
1578                                 continuation, withinRange);
1579                             
1580                             m_out.appendTo(withinRange, continuation);
1581                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
1582                             m_out.jump(continuation);
1583                             
1584                             m_out.appendTo(continuation, lastNext);
1585                             intValue = m_out.phi(m_out.int32, intValues);
1586                         } else if (isSigned(type))
1587                             intValue = doubleToInt32(doubleValue);
1588                         else
1589                             intValue = doubleToUInt32(doubleValue);
1590                         break;
1591                     }
1592                         
1593                     default:
1594                         RELEASE_ASSERT_NOT_REACHED();
1595                     }
1596                     
1597                     switch (elementSize(type)) {
1598                     case 1:
1599                         m_out.store8(m_out.intCast(intValue, m_out.int8), pointer);
1600                         break;
1601                     case 2:
1602                         m_out.store16(m_out.intCast(intValue, m_out.int16), pointer);
1603                         break;
1604                     case 4:
1605                         m_out.store32(intValue, pointer);
1606                         break;
1607                     default:
1608                         RELEASE_ASSERT_NOT_REACHED();
1609                     }
1610                     
1611                     return;
1612                 }
1613                 
1614                 ASSERT(isFloat(type));
1615                 
1616                 LValue value = lowDouble(child3);
1617                 switch (type) {
1618                 case TypeFloat32:
1619                     m_out.storeFloat(m_out.fpCast(value, m_out.floatType), pointer);
1620                     break;
1621                 case TypeFloat64:
1622                     m_out.storeDouble(value, pointer);
1623                     break;
1624                 default:
1625                     RELEASE_ASSERT_NOT_REACHED();
1626                 }
1627                 return;
1628             }
1629             
1630             RELEASE_ASSERT_NOT_REACHED();
1631             break;
1632         }
1633     }
1634     
1635     void compileGetByOffset()
1636     {
1637         StorageAccessData& data =
1638             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1639         
1640         setJSValue(
1641             m_out.load64(
1642                 m_out.address(
1643                     m_heaps.properties[data.identifierNumber],
1644                     lowStorage(m_node->child1()),
1645                     offsetRelativeToBase(data.offset))));
1646     }
1647     
1648     void compilePutByOffset()
1649     {
1650         StorageAccessData& data =
1651             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1652         
1653         m_out.store64(
1654             lowJSValue(m_node->child3()),
1655             m_out.address(
1656                 m_heaps.properties[data.identifierNumber],
1657                 lowStorage(m_node->child1()),
1658                 offsetRelativeToBase(data.offset)));
1659     }
1660     
1661     void compileGetGlobalVar()
1662     {
1663         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
1664     }
1665     
1666     void compilePutGlobalVar()
1667     {
1668         m_out.store64(
1669             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
1670     }
1671     
1672     void compileGlobalVarWatchpoint()
1673     {
1674         // FIXME: Implement watchpoints.
1675         // https://bugs.webkit.org/show_bug.cgi?id=113647
1676     }
1677     
1678     void compileGetMyScope()
1679     {
1680         setJSValue(m_out.loadPtr(addressFor(
1681             m_node->codeOrigin.stackOffset() + JSStack::ScopeChain)));
1682     }
1683     
1684     void compileSkipScope()
1685     {
1686         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
1687     }
1688     
1689     void compileGetClosureRegisters()
1690     {
1691         setStorage(m_out.loadPtr(
1692             lowCell(m_node->child1()), m_heaps.JSVariableObject_registers));
1693     }
1694     
1695     void compileGetClosureVar()
1696     {
1697         setJSValue(m_out.load64(
1698             addressFor(lowStorage(m_node->child1()), m_node->varNumber())));
1699     }
1700     
1701     void compilePutClosureVar()
1702     {
1703         m_out.store64(
1704             lowJSValue(m_node->child3()),
1705             addressFor(lowStorage(m_node->child2()), m_node->varNumber()));
1706     }
1707     
1708     void compileCompareEq()
1709     {
1710         if (m_node->isBinaryUseKind(Int32Use)
1711             || m_node->isBinaryUseKind(NumberUse)
1712             || m_node->isBinaryUseKind(ObjectUse)) {
1713             compileCompareStrictEq();
1714             return;
1715         }
1716         
1717         RELEASE_ASSERT_NOT_REACHED();
1718     }
1719     
1720     void compileCompareEqConstant()
1721     {
1722         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
1723         masqueradesAsUndefinedWatchpointIfIsStillValid();
1724         setBoolean(
1725             equalNullOrUndefined(
1726                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
1727     }
1728     
1729     void compileCompareStrictEq()
1730     {
1731         if (m_node->isBinaryUseKind(Int32Use)) {
1732             setBoolean(
1733                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1734             return;
1735         }
1736         
1737         if (m_node->isBinaryUseKind(NumberUse)) {
1738             setBoolean(
1739                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1740             return;
1741         }
1742         
1743         if (m_node->isBinaryUseKind(ObjectUse)) {
1744             masqueradesAsUndefinedWatchpointIfIsStillValid();
1745             setBoolean(
1746                 m_out.equal(
1747                     lowNonNullObject(m_node->child1()),
1748                     lowNonNullObject(m_node->child2())));
1749             return;
1750         }
1751         
1752         RELEASE_ASSERT_NOT_REACHED();
1753     }
1754     
1755     void compileCompareStrictEqConstant()
1756     {
1757         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
1758
1759         if (constant.isUndefinedOrNull()
1760             && !masqueradesAsUndefinedWatchpointIfIsStillValid()) {
1761             if (constant.isNull()) {
1762                 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull));
1763                 return;
1764             }
1765         
1766             ASSERT(constant.isUndefined());
1767             setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
1768             return;
1769         }
1770         
1771         setBoolean(
1772             m_out.equal(
1773                 lowJSValue(m_node->child1()),
1774                 m_out.constInt64(JSValue::encode(constant))));
1775     }
1776     
1777     void compileCompareLess()
1778     {
1779         if (m_node->isBinaryUseKind(Int32Use)) {
1780             setBoolean(
1781                 m_out.lessThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1782             return;
1783         }
1784         
1785         if (m_node->isBinaryUseKind(NumberUse)) {
1786             setBoolean(
1787                 m_out.doubleLessThan(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1788             return;
1789         }
1790         
1791         RELEASE_ASSERT_NOT_REACHED();
1792     }
1793     
1794     void compileCompareLessEq()
1795     {
1796         if (m_node->isBinaryUseKind(Int32Use)) {
1797             setBoolean(
1798                 m_out.lessThanOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1799             return;
1800         }
1801         
1802         if (m_node->isBinaryUseKind(NumberUse)) {
1803             setBoolean(
1804                 m_out.doubleLessThanOrEqual(
1805                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1806             return;
1807         }
1808         
1809         RELEASE_ASSERT_NOT_REACHED();
1810     }
1811     
1812     void compileCompareGreater()
1813     {
1814         if (m_node->isBinaryUseKind(Int32Use)) {
1815             setBoolean(
1816                 m_out.greaterThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1817             return;
1818         }
1819         
1820         if (m_node->isBinaryUseKind(NumberUse)) {
1821             setBoolean(
1822                 m_out.doubleGreaterThan(
1823                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1824             return;
1825         }
1826         
1827         RELEASE_ASSERT_NOT_REACHED();
1828     }
1829     
1830     void compileCompareGreaterEq()
1831     {
1832         if (m_node->isBinaryUseKind(Int32Use)) {
1833             setBoolean(
1834                 m_out.greaterThanOrEqual(
1835                     lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1836             return;
1837         }
1838         
1839         if (m_node->isBinaryUseKind(NumberUse)) {
1840             setBoolean(
1841                 m_out.doubleGreaterThanOrEqual(
1842                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1843             return;
1844         }
1845         
1846         RELEASE_ASSERT_NOT_REACHED();
1847     }
1848     
1849     void compileLogicalNot()
1850     {
1851         setBoolean(m_out.bitNot(boolify(m_node->child1())));
1852     }
1853     
1854     void compileCallOrConstruct()
1855     {
1856         // FIXME: This is unacceptably slow.
1857         // https://bugs.webkit.org/show_bug.cgi?id=113621
1858         
1859         J_DFGOperation_E function =
1860             m_node->op() == Call ? operationFTLCall : operationFTLConstruct;
1861         
1862         int dummyThisArgument = m_node->op() == Call ? 0 : 1;
1863         
1864         int numPassedArgs = m_node->numChildren() - 1;
1865         
1866         LValue calleeFrame = m_out.add(
1867             m_callFrame,
1868             m_out.constIntPtr(sizeof(Register) * localToOperand(codeBlock()->m_numCalleeRegisters)));
1869         
1870         m_out.store32(
1871             m_out.constInt32(numPassedArgs + dummyThisArgument),
1872             payloadFor(calleeFrame, JSStack::ArgumentCount));
1873         m_out.store64(m_callFrame, addressFor(calleeFrame, JSStack::CallerFrame));
1874         m_out.store64(
1875             lowJSValue(m_graph.varArgChild(m_node, 0)),
1876             addressFor(calleeFrame, JSStack::Callee));
1877         
1878         for (int i = 0; i < numPassedArgs; ++i) {
1879             m_out.store64(
1880                 lowJSValue(m_graph.varArgChild(m_node, 1 + i)),
1881                 addressFor(calleeFrame, argumentToOperand(i + dummyThisArgument)));
1882         }
1883         
1884         setJSValue(vmCall(m_out.operation(function), calleeFrame));
1885     }
1886     
1887     void compileJump()
1888     {
1889         m_out.jump(lowBlock(m_node->takenBlock()));
1890     }
1891     
1892     void compileBranch()
1893     {
1894         m_out.branch(
1895             boolify(m_node->child1()),
1896             lowBlock(m_node->takenBlock()),
1897             lowBlock(m_node->notTakenBlock()));
1898     }
1899     
1900     void compileSwitch()
1901     {
1902         SwitchData* data = m_node->switchData();
1903         switch (data->kind) {
1904         case SwitchImm: {
1905             Vector<ValueFromBlock, 2> intValues;
1906             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
1907             
1908             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
1909             
1910             switch (m_node->child1().useKind()) {
1911             case Int32Use: {
1912                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
1913                 m_out.jump(switchOnInts);
1914                 break;
1915             }
1916                 
1917             case UntypedUse: {
1918                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
1919                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
1920                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
1921                 
1922                 LValue boxedValue = lowJSValue(m_node->child1());
1923                 m_out.branch(isNotInt32(boxedValue), isNotInt, isInt);
1924                 
1925                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
1926                 
1927                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
1928                 m_out.jump(switchOnInts);
1929                 
1930                 m_out.appendTo(isNotInt, isDouble);
1931                 m_out.branch(
1932                     isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble);
1933                 
1934                 m_out.appendTo(isDouble, innerLastNext);
1935                 LValue doubleValue = unboxDouble(boxedValue);
1936                 LValue intInDouble = m_out.fpToInt32(doubleValue);
1937                 intValues.append(m_out.anchor(intInDouble));
1938                 m_out.branch(
1939                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
1940                     switchOnInts, lowBlock(data->fallThrough));
1941                 break;
1942             }
1943                 
1944             default:
1945                 RELEASE_ASSERT_NOT_REACHED();
1946                 break;
1947             }
1948             
1949             m_out.appendTo(switchOnInts, lastNext);
1950             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
1951             return;
1952         }
1953         
1954         case SwitchChar: {
1955             LValue stringValue;
1956             
1957             switch (m_node->child1().useKind()) {
1958             case StringUse: {
1959                 stringValue = lowString(m_node->child1());
1960                 break;
1961             }
1962                 
1963             case UntypedUse: {
1964                 LValue unboxedValue = lowJSValue(m_node->child1());
1965                 
1966                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
1967                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
1968                 
1969                 m_out.branch(
1970                     isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase);
1971                 
1972                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
1973                 LValue cellValue = unboxedValue;
1974                 m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase);
1975                 
1976                 m_out.appendTo(isStringCase, lastNext);
1977                 stringValue = cellValue;
1978                 break;
1979             }
1980                 
1981             default:
1982                 RELEASE_ASSERT_NOT_REACHED();
1983                 break;
1984             }
1985             
1986             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
1987             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
1988             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
1989             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
1990             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
1991             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
1992             
1993             m_out.branch(
1994                 m_out.notEqual(
1995                     m_out.load32(stringValue, m_heaps.JSString_length),
1996                     m_out.int32One),
1997                 lowBlock(data->fallThrough), lengthIs1);
1998             
1999             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
2000             Vector<ValueFromBlock, 2> values;
2001             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
2002             values.append(m_out.anchor(fastValue));
2003             m_out.branch(m_out.isNull(fastValue), needResolution, resolved);
2004             
2005             m_out.appendTo(needResolution, resolved);
2006             values.append(m_out.anchor(
2007                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
2008             m_out.jump(resolved);
2009             
2010             m_out.appendTo(resolved, is8Bit);
2011             LValue value = m_out.phi(m_out.intPtr, values);
2012             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
2013             m_out.branch(
2014                 m_out.testNonZero32(
2015                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
2016                     m_out.constInt32(StringImpl::flagIs8Bit())),
2017                 is8Bit, is16Bit);
2018             
2019             Vector<ValueFromBlock, 2> characters;
2020             m_out.appendTo(is8Bit, is16Bit);
2021             characters.append(m_out.anchor(
2022                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
2023             m_out.jump(continuation);
2024             
2025             m_out.appendTo(is16Bit, continuation);
2026             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
2027             m_out.jump(continuation);
2028             
2029             m_out.appendTo(continuation, lastNext);
2030             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
2031             return;
2032         }
2033         
2034         case SwitchString:
2035             RELEASE_ASSERT_NOT_REACHED();
2036             break;
2037         }
2038         
2039         RELEASE_ASSERT_NOT_REACHED();
2040     }
2041     
2042     void compileReturn()
2043     {
2044         // FIXME: have a real epilogue when we switch to using our calling convention.
2045         // https://bugs.webkit.org/show_bug.cgi?id=113621
2046         m_out.ret(lowJSValue(m_node->child1()));
2047     }
2048     
2049     void compileForceOSRExit()
2050     {
2051         terminate(InadequateCoverage);
2052     }
2053     
2054     LValue boolify(Edge edge)
2055     {
2056         switch (edge.useKind()) {
2057         case BooleanUse:
2058             return lowBoolean(m_node->child1());
2059         case Int32Use:
2060             return m_out.notZero32(lowInt32(m_node->child1()));
2061         case NumberUse:
2062             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
2063         case ObjectOrOtherUse:
2064             return m_out.bitNot(
2065                 equalNullOrUndefined(
2066                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
2067                     ManualOperandSpeculation));
2068         default:
2069             RELEASE_ASSERT_NOT_REACHED();
2070             return 0;
2071         }
2072     }
2073     
2074     enum StringOrObjectMode {
2075         AllCellsAreFalse,
2076         CellCaseSpeculatesObject
2077     };
2078     enum EqualNullOrUndefinedMode {
2079         EqualNull,
2080         EqualUndefined,
2081         EqualNullOrUndefined,
2082         SpeculateNullOrUndefined
2083     };
2084     LValue equalNullOrUndefined(
2085         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
2086         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
2087     {
2088         bool validWatchpoint = masqueradesAsUndefinedWatchpointIfIsStillValid();
2089         
2090         LValue value = lowJSValue(edge, operandMode);
2091         
2092         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
2093         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
2094         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
2095         
2096         m_out.branch(isNotCell(value), primitiveCase, cellCase);
2097         
2098         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
2099         
2100         Vector<ValueFromBlock, 3> results;
2101         
2102         switch (cellMode) {
2103         case AllCellsAreFalse:
2104             break;
2105         case CellCaseSpeculatesObject:
2106             FTL_TYPE_CHECK(
2107                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
2108                 m_out.equal(
2109                     m_out.loadPtr(value, m_heaps.JSCell_structure),
2110                     m_out.constIntPtr(vm().stringStructure.get())));
2111             break;
2112         }
2113         
2114         if (validWatchpoint) {
2115             results.append(m_out.anchor(m_out.booleanFalse));
2116             m_out.jump(continuation);
2117         } else {
2118             LBasicBlock masqueradesCase =
2119                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
2120                 
2121             LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure);
2122             
2123             results.append(m_out.anchor(m_out.booleanFalse));
2124             
2125             m_out.branch(
2126                 m_out.testNonZero8(
2127                     m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2128                     m_out.constInt8(MasqueradesAsUndefined)),
2129                 masqueradesCase, continuation);
2130             
2131             m_out.appendTo(masqueradesCase, primitiveCase);
2132             
2133             results.append(m_out.anchor(
2134                 m_out.equal(
2135                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)),
2136                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
2137             m_out.jump(continuation);
2138         }
2139         
2140         m_out.appendTo(primitiveCase, continuation);
2141         
2142         LValue primitiveResult;
2143         switch (primitiveMode) {
2144         case EqualNull:
2145             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
2146             break;
2147         case EqualUndefined:
2148             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
2149             break;
2150         case EqualNullOrUndefined:
2151             primitiveResult = m_out.equal(
2152                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
2153                 m_out.constInt64(ValueNull));
2154             break;
2155         case SpeculateNullOrUndefined:
2156             FTL_TYPE_CHECK(
2157                 jsValueValue(value), edge, SpecCell | SpecOther,
2158                 m_out.notEqual(
2159                     m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
2160                     m_out.constInt64(ValueNull)));
2161             primitiveResult = m_out.booleanTrue;
2162             break;
2163         }
2164         results.append(m_out.anchor(primitiveResult));
2165         m_out.jump(continuation);
2166         
2167         m_out.appendTo(continuation, lastNext);
2168         
2169         return m_out.phi(m_out.boolean, results);
2170     }
2171     
2172     template<typename FunctionType>
2173     void contiguousPutByValOutOfBounds(
2174         FunctionType slowPathFunction,
2175         LValue base, LValue storage, LValue index, LValue value,
2176         LBasicBlock continuation)
2177     {
2178         LValue isNotInBounds = m_out.aboveOrEqual(
2179             index, m_out.load32(storage, m_heaps.Butterfly_publicLength));
2180         if (m_node->arrayMode().isInBounds())
2181             speculate(StoreToHoleOrOutOfBounds, noValue(), 0, isNotInBounds);
2182         else {
2183             LBasicBlock notInBoundsCase =
2184                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
2185             LBasicBlock performStore =
2186                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
2187                 
2188             m_out.branch(isNotInBounds, notInBoundsCase, performStore);
2189                 
2190             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
2191                 
2192             LValue isOutOfBounds = m_out.aboveOrEqual(
2193                 index, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
2194                 
2195             if (!m_node->arrayMode().isOutOfBounds())
2196                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
2197             else {
2198                 LBasicBlock outOfBoundsCase =
2199                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
2200                 LBasicBlock holeCase =
2201                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
2202                     
2203                 m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase);
2204                     
2205                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
2206                     
2207                 vmCall(
2208                     m_out.operation(slowPathFunction),
2209                     m_callFrame, base, index, value);
2210                     
2211                 m_out.jump(continuation);
2212                     
2213                 m_out.appendTo(holeCase, innerLastNext);
2214             }
2215             
2216             m_out.store32(
2217                 m_out.add(index, m_out.int32One),
2218                 storage, m_heaps.Butterfly_publicLength);
2219                 
2220             m_out.jump(performStore);
2221             m_out.appendTo(performStore, lastNext);
2222         }
2223     }
2224     
2225     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
2226     {
2227         Vector<SwitchCase> cases;
2228         for (unsigned i = 0; i < data->cases.size(); ++i) {
2229             cases.append(SwitchCase(
2230                 constInt(type, data->cases[i].value.switchLookupValue()),
2231                 lowBlock(data->cases[i].target)));
2232         }
2233         
2234         m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough));
2235     }
2236     
2237     LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
2238     {
2239         // FIXME: Optimize double-to-int conversions.
2240         // <rdar://problem/14938465>
2241         
2242         LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
2243         LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
2244         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
2245         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
2246         
2247         Vector<ValueFromBlock, 2> results;
2248         
2249         m_out.branch(
2250             m_out.greaterThanOrEqual(doubleValue, m_out.constDouble(low)),
2251             greatEnough, slowPath);
2252         
2253         LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
2254         m_out.branch(
2255             m_out.lessThanOrEqual(doubleValue, m_out.constDouble(high)),
2256             withinRange, slowPath);
2257         
2258         m_out.appendTo(withinRange, slowPath);
2259         LValue fastResult;
2260         if (isSigned)
2261             fastResult = m_out.fpToInt32(doubleValue);
2262         else
2263             fastResult = m_out.fpToUInt32(doubleValue);
2264         results.append(m_out.anchor(fastResult));
2265         m_out.jump(continuation);
2266         
2267         m_out.appendTo(slowPath, continuation);
2268         results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
2269         m_out.jump(continuation);
2270         
2271         m_out.appendTo(continuation, lastNext);
2272         return m_out.phi(m_out.int32, results);
2273     }
2274     
2275     LValue doubleToInt32(LValue doubleValue)
2276     {
2277         double limit = pow(2, 31) - 1;
2278         return doubleToInt32(doubleValue, -limit, limit);
2279     }
2280     
2281     LValue doubleToUInt32(LValue doubleValue)
2282     {
2283         return doubleToInt32(doubleValue, 0, pow(2, 32) - 1, false);
2284     }
2285     
2286     void speculateBackward(
2287         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
2288     {
2289         appendOSRExit(
2290             kind, lowValue, highValue, failCondition, BackwardSpeculation, FormattedValue());
2291     }
2292     
2293     void speculateForward(
2294         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2295         const FormattedValue& recovery)
2296     {
2297         appendOSRExit(
2298             kind, lowValue, highValue, failCondition, ForwardSpeculation, recovery);
2299     }
2300     
2301     void speculate(
2302         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
2303     {
2304         appendOSRExit(
2305             kind, lowValue, highValue, failCondition, m_direction, FormattedValue());
2306     }
2307     
2308     void terminate(ExitKind kind)
2309     {
2310         speculate(kind, noValue(), 0, m_out.booleanTrue);
2311     }
2312     
2313     void backwardTypeCheck(
2314         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2315         LValue failCondition)
2316     {
2317         appendTypeCheck(
2318             lowValue, highValue, typesPassedThrough, failCondition, BackwardSpeculation,
2319             FormattedValue());
2320     }
2321     
2322     void forwardTypeCheck(
2323         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2324         LValue failCondition)
2325     {
2326         appendTypeCheck(
2327             lowValue, highValue, typesPassedThrough, failCondition, ForwardSpeculation,
2328             FormattedValue());
2329     }
2330     
2331     void typeCheck(
2332         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2333         LValue failCondition)
2334     {
2335         appendTypeCheck(
2336             lowValue, highValue, typesPassedThrough, failCondition, m_direction,
2337             FormattedValue());
2338     }
2339     
2340     void appendTypeCheck(
2341         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2342         LValue failCondition, SpeculationDirection direction, FormattedValue recovery)
2343     {
2344         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
2345             return;
2346         ASSERT(mayHaveTypeCheck(highValue.useKind()));
2347         appendOSRExit(BadType, lowValue, highValue.node(), failCondition, direction, recovery);
2348         m_interpreter.filter(highValue, typesPassedThrough);
2349     }
2350     
2351     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2352     {
2353         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
2354         
2355         LoweredNodeValue value = m_int32Values.get(edge.node());
2356         if (isValid(value))
2357             return value.value();
2358         
2359         value = m_jsValueValues.get(edge.node());
2360         if (isValid(value)) {
2361             LValue boxedResult = value.value();
2362             FTL_TYPE_CHECK(
2363                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
2364             LValue result = unboxInt32(boxedResult);
2365             setInt32(edge.node(), result);
2366             return result;
2367         }
2368
2369         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
2370         terminate(Uncountable);
2371         return m_out.int32Zero;
2372     }
2373     
2374     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2375     {
2376         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
2377         
2378         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2379         if (isValid(value)) {
2380             LValue uncheckedValue = value.value();
2381             FTL_TYPE_CHECK(
2382                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
2383             return uncheckedValue;
2384         }
2385         
2386         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
2387         terminate(Uncountable);
2388         return m_out.intPtrZero;
2389     }
2390     
2391     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2392     {
2393         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2394         
2395         LValue result = lowCell(edge, mode);
2396         speculateObject(edge, result);
2397         return result;
2398     }
2399     
2400     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2401     {
2402         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse);
2403         
2404         LValue result = lowCell(edge, mode);
2405         speculateString(edge, result);
2406         return result;
2407     }
2408     
2409     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2410     {
2411         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2412         
2413         LValue result = lowCell(edge, mode);
2414         speculateNonNullObject(edge, result);
2415         return result;
2416     }
2417     
2418     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2419     {
2420         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
2421         
2422         LoweredNodeValue value = m_booleanValues.get(edge.node());
2423         if (isValid(value))
2424             return value.value();
2425         
2426         value = m_jsValueValues.get(edge.node());
2427         if (isValid(value)) {
2428             LValue unboxedResult = value.value();
2429             FTL_TYPE_CHECK(
2430                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
2431             LValue result = unboxBoolean(unboxedResult);
2432             setBoolean(edge.node(), result);
2433             return result;
2434         }
2435         
2436         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
2437         terminate(Uncountable);
2438         return m_out.booleanFalse;
2439     }
2440     
2441     LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2442     {
2443         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind()));
2444         
2445         LoweredNodeValue value = m_doubleValues.get(edge.node());
2446         if (isValid(value))
2447             return value.value();
2448         
2449         value = m_int32Values.get(edge.node());
2450         if (isValid(value)) {
2451             LValue result = m_out.intToDouble(value.value());
2452             setDouble(edge.node(), result);
2453             return result;
2454         }
2455         
2456         value = m_jsValueValues.get(edge.node());
2457         if (isValid(value)) {
2458             LValue boxedResult = value.value();
2459             
2460             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case"));
2461             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case"));
2462             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation"));
2463             
2464             m_out.branch(isNotInt32(boxedResult), doubleCase, intCase);
2465             
2466             LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
2467             
2468             ValueFromBlock intToDouble = m_out.anchor(
2469                 m_out.intToDouble(unboxInt32(boxedResult)));
2470             m_out.jump(continuation);
2471             
2472             m_out.appendTo(doubleCase, continuation);
2473             
2474             FTL_TYPE_CHECK(
2475                 jsValueValue(boxedResult), edge, SpecNumber, isCellOrMisc(boxedResult));
2476             
2477             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult));
2478             m_out.jump(continuation);
2479             
2480             m_out.appendTo(continuation, lastNext);
2481             
2482             LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
2483             
2484             setDouble(edge.node(), result);
2485             return result;
2486         }
2487         
2488         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecNumber));
2489         terminate(Uncountable);
2490         return m_out.doubleZero;
2491     }
2492     
2493     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2494     {
2495         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2496         
2497         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2498         if (isValid(value))
2499             return value.value();
2500         
2501         value = m_int32Values.get(edge.node());
2502         if (isValid(value)) {
2503             LValue result = boxInt32(value.value());
2504             setJSValue(edge.node(), result);
2505             return result;
2506         }
2507         
2508         value = m_booleanValues.get(edge.node());
2509         if (isValid(value)) {
2510             LValue result = boxBoolean(value.value());
2511             setJSValue(edge.node(), result);
2512             return result;
2513         }
2514         
2515         value = m_doubleValues.get(edge.node());
2516         if (isValid(value)) {
2517             LValue result = boxDouble(value.value());
2518             setJSValue(edge.node(), result);
2519             return result;
2520         }
2521         
2522         RELEASE_ASSERT_NOT_REACHED();
2523         return 0;
2524     }
2525     
2526     LValue lowStorage(Edge edge)
2527     {
2528         LoweredNodeValue value = m_storageValues.get(edge.node());
2529         if (isValid(value))
2530             return value.value();
2531         
2532         LValue result = lowCell(edge);
2533         setStorage(edge.node(), result);
2534         return result;
2535     }
2536     
2537     LValue isNotInt32(LValue jsValue)
2538     {
2539         return m_out.below(jsValue, m_tagTypeNumber);
2540     }
2541     LValue unboxInt32(LValue jsValue)
2542     {
2543         return m_out.castToInt32(jsValue);
2544     }
2545     LValue boxInt32(LValue value)
2546     {
2547         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
2548     }
2549     
2550     LValue isCellOrMisc(LValue jsValue)
2551     {
2552         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
2553     }
2554     LValue unboxDouble(LValue jsValue)
2555     {
2556         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
2557     }
2558     LValue boxDouble(LValue doubleValue)
2559     {
2560         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
2561     }
2562     
2563     LValue isNotCell(LValue jsValue)
2564     {
2565         return m_out.testNonZero64(jsValue, m_tagMask);
2566     }
2567     
2568     LValue isNotBoolean(LValue jsValue)
2569     {
2570         return m_out.testNonZero64(
2571             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
2572             m_out.constInt64(~1));
2573     }
2574     LValue unboxBoolean(LValue jsValue)
2575     {
2576         // We want to use a cast that guarantees that LLVM knows that even the integer
2577         // value is just 0 or 1. But for now we do it the dumb way.
2578         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
2579     }
2580     LValue boxBoolean(LValue value)
2581     {
2582         return m_out.select(
2583             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
2584     }
2585
2586     void speculate(Edge edge)
2587     {
2588         switch (edge.useKind()) {
2589         case UntypedUse:
2590             break;
2591         case KnownInt32Use:
2592         case KnownNumberUse:
2593             ASSERT(!m_interpreter.needsTypeCheck(edge));
2594             break;
2595         case Int32Use:
2596             speculateInt32(edge);
2597             break;
2598         case CellUse:
2599             speculateCell(edge);
2600             break;
2601         case KnownCellUse:
2602             ASSERT(!m_interpreter.needsTypeCheck(edge));
2603             break;
2604         case ObjectUse:
2605             speculateObject(edge);
2606             break;
2607         case ObjectOrOtherUse:
2608             speculateObjectOrOther(edge);
2609             break;
2610         case StringUse:
2611             speculateString(edge);
2612             break;
2613         case RealNumberUse:
2614             speculateRealNumber(edge);
2615             break;
2616         case NumberUse:
2617             speculateNumber(edge);
2618             break;
2619         case BooleanUse:
2620             speculateBoolean(edge);
2621             break;
2622         default:
2623             dataLog("Unsupported speculation use kind: ", edge.useKind(), "\n");
2624             RELEASE_ASSERT_NOT_REACHED();
2625         }
2626     }
2627     
2628     void speculate(Node*, Edge edge)
2629     {
2630         speculate(edge);
2631     }
2632     
2633     void speculateInt32(Edge edge)
2634     {
2635         lowInt32(edge);
2636     }
2637     
2638     void speculateCell(Edge edge)
2639     {
2640         lowCell(edge);
2641     }
2642     
2643     LValue isObject(LValue cell)
2644     {
2645         return m_out.notEqual(
2646             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2647             m_out.constIntPtr(vm().stringStructure.get()));
2648     }
2649     
2650     LValue isNotString(LValue cell)
2651     {
2652         return isObject(cell);
2653     }
2654     
2655     LValue isString(LValue cell)
2656     {
2657         return m_out.equal(
2658             m_out.loadPtr(cell, m_heaps.JSCell_structure),
2659             m_out.constIntPtr(vm().stringStructure.get()));
2660     }
2661     
2662     LValue isNotObject(LValue cell)
2663     {
2664         return isString(cell);
2665     }
2666     
2667     LValue isArrayType(LValue cell, ArrayMode arrayMode)
2668     {
2669         switch (arrayMode.type()) {
2670         case Array::Int32:
2671         case Array::Double:
2672         case Array::Contiguous: {
2673             LValue indexingType = m_out.load8(
2674                 m_out.loadPtr(cell, m_heaps.JSCell_structure),
2675                 m_heaps.Structure_indexingType);
2676             
2677             switch (arrayMode.arrayClass()) {
2678             case Array::OriginalArray:
2679                 RELEASE_ASSERT_NOT_REACHED();
2680                 return 0;
2681                 
2682             case Array::Array:
2683                 return m_out.equal(
2684                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
2685                     m_out.constInt8(IsArray | arrayMode.shapeMask()));
2686                 
2687             default:
2688                 return m_out.equal(
2689                     m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
2690                     m_out.constInt8(arrayMode.shapeMask()));
2691             }
2692         }
2693             
2694         default:
2695             return hasClassInfo(cell, classInfoForType(arrayMode.typedArrayType()));
2696         }
2697     }
2698     
2699     LValue hasClassInfo(LValue cell, const ClassInfo* classInfo)
2700     {
2701         return m_out.equal(
2702             m_out.loadPtr(
2703                 m_out.loadPtr(cell, m_heaps.JSCell_structure),
2704                 m_heaps.Structure_classInfo),
2705             m_out.constIntPtr(classInfo));
2706     }
2707     
2708     void speculateObject(Edge edge, LValue cell)
2709     {
2710         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
2711     }
2712     
2713     void speculateObject(Edge edge)
2714     {
2715         speculateObject(edge, lowCell(edge));
2716     }
2717     
2718     void speculateObjectOrOther(Edge edge)
2719     {
2720         if (!m_interpreter.needsTypeCheck(edge))
2721             return;
2722         
2723         LValue value = lowJSValue(edge);
2724         
2725         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
2726         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
2727         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
2728         
2729         m_out.branch(isNotCell(value), primitiveCase, cellCase);
2730         
2731         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
2732         
2733         FTL_TYPE_CHECK(
2734             jsValueValue(value), edge, (~SpecCell) | SpecObject,
2735             m_out.equal(
2736                 m_out.loadPtr(value, m_heaps.JSCell_structure),
2737                 m_out.constIntPtr(vm().stringStructure.get())));
2738         
2739         m_out.jump(continuation);
2740         
2741         m_out.appendTo(primitiveCase, continuation);
2742         
2743         FTL_TYPE_CHECK(
2744             jsValueValue(value), edge, SpecCell | SpecOther,
2745             m_out.notEqual(
2746                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
2747                 m_out.constInt64(ValueNull)));
2748         
2749         m_out.jump(continuation);
2750         
2751         m_out.appendTo(continuation, lastNext);
2752     }
2753     
2754     void speculateString(Edge edge, LValue cell)
2755     {
2756         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
2757     }
2758     
2759     void speculateString(Edge edge)
2760     {
2761         speculateString(edge, lowCell(edge));
2762     }
2763     
2764     void speculateNonNullObject(Edge edge, LValue cell)
2765     {
2766         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
2767         FTL_TYPE_CHECK(
2768             jsValueValue(cell), edge, SpecObject, 
2769             m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get())));
2770         if (masqueradesAsUndefinedWatchpointIfIsStillValid())
2771             return;
2772         
2773         speculate(
2774             BadType, jsValueValue(cell), edge.node(),
2775             m_out.testNonZero8(
2776                 m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2777                 m_out.constInt8(MasqueradesAsUndefined)));
2778     }
2779     
2780     void speculateNumber(Edge edge)
2781     {
2782         // Do an early return here because lowDouble() can create a lot of control flow.
2783         if (!m_interpreter.needsTypeCheck(edge))
2784             return;
2785         
2786         lowDouble(edge);
2787     }
2788     
2789     void speculateRealNumber(Edge edge)
2790     {
2791         // Do an early return here because lowDouble() can create a lot of control flow.
2792         if (!m_interpreter.needsTypeCheck(edge))
2793             return;
2794         
2795         LValue value = lowDouble(edge);
2796         FTL_TYPE_CHECK(
2797             doubleValue(value), edge, SpecRealNumber,
2798             m_out.doubleNotEqualOrUnordered(value, value));
2799     }
2800     
2801     void speculateBoolean(Edge edge)
2802     {
2803         lowBoolean(edge);
2804     }
2805
2806     bool masqueradesAsUndefinedWatchpointIsStillValid()
2807     {
2808         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin);
2809     }
2810     
2811     bool masqueradesAsUndefinedWatchpointIfIsStillValid()
2812     {
2813         if (!masqueradesAsUndefinedWatchpointIsStillValid())
2814             return false;
2815         
2816         // FIXME: Implement masquerades-as-undefined watchpoints.
2817         // https://bugs.webkit.org/show_bug.cgi?id=113647
2818         return true;
2819     }
2820     
2821     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
2822     
2823     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
2824     {
2825         callPreflight();
2826         LValue result = m_out.call(function);
2827         callCheck(mode);
2828         return result;
2829     }
2830     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
2831     {
2832         callPreflight();
2833         LValue result = m_out.call(function, arg1);
2834         callCheck(mode);
2835         return result;
2836     }
2837     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
2838     {
2839         callPreflight();
2840         LValue result = m_out.call(function, arg1, arg2);
2841         callCheck(mode);
2842         return result;
2843     }
2844     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
2845     {
2846         callPreflight();
2847         LValue result = m_out.call(function, arg1, arg2, arg3);
2848         callCheck(mode);
2849         return result;
2850     }
2851     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
2852     {
2853         callPreflight();
2854         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
2855         callCheck(mode);
2856         return result;
2857     }
2858     
2859     void callPreflight(CodeOrigin codeOrigin)
2860     {
2861         m_out.store32(
2862             m_out.constInt32(
2863                 CallFrame::Location::encodeAsCodeOriginIndex(
2864                     codeBlock()->addCodeOrigin(codeOrigin))), 
2865             tagFor(JSStack::ArgumentCount));
2866     }
2867     void callPreflight()
2868     {
2869         callPreflight(m_node->codeOrigin);
2870     }
2871     
2872     void callCheck(ExceptionCheckMode mode = CheckExceptions)
2873     {
2874         if (mode == NoExceptions)
2875             return;
2876         
2877         LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception"));
2878         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
2879         
2880         m_out.branch(
2881             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
2882             didHaveException, continuation);
2883         
2884         LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation);
2885         // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622
2886         m_out.crash();
2887         
2888         m_out.appendTo(continuation, lastNext);
2889     }
2890     
2891     bool isLive(Node* node)
2892     {
2893         return m_live.contains(node);
2894     }
2895     
2896     void use(Edge edge)
2897     {
2898         ASSERT(edge->hasResult());
2899         if (!edge.doesKill())
2900             return;
2901         m_live.remove(edge.node());
2902     }
2903     
2904     // Wrapper used only for DFG_NODE_DO_TO_CHILDREN
2905     void use(Node*, Edge edge)
2906     {
2907         use(edge);
2908     }
2909     
2910     LBasicBlock lowBlock(BasicBlock* block)
2911     {
2912         return m_blocks.get(block);
2913     }
2914     
2915     void initializeOSRExitStateForBlock()
2916     {
2917         for (unsigned i = m_valueSources.size(); i--;) {
2918             FlushFormat format = m_highBlock->ssa->flushFormatAtHead[i];
2919             switch (format) {
2920             case DeadFlush: {
2921                 // Must consider available nodes instead.
2922                 Node* node = m_highBlock->ssa->availabilityAtHead[i];
2923                 if (!node) {
2924                     m_valueSources[i] = ValueSource(SourceIsDead);
2925                     break;
2926                 }
2927                 
2928                 m_valueSources[i] = ValueSource(node);
2929                 break;
2930             }
2931                 
2932             case FlushedInt32:
2933                 m_valueSources[i] = ValueSource(Int32InJSStack);
2934                 break;
2935                 
2936             case FlushedDouble:
2937                 m_valueSources[i] = ValueSource(DoubleInJSStack);
2938                 break;
2939                 
2940             case FlushedCell:
2941             case FlushedBoolean:
2942             case FlushedJSValue:
2943                 m_valueSources[i] = ValueSource(ValueInJSStack);
2944                 break;
2945             }
2946         }
2947     }
2948     
2949     void appendOSRExit(
2950         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2951         SpeculationDirection direction, FormattedValue recovery)
2952     {
2953         if (Options::ftlTrapsOnOSRExit()) {
2954             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2955             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2956             
2957             m_out.branch(failCondition, failCase, continuation);
2958             
2959             LBasicBlock lastNext = m_out.appendTo(failCase, continuation);
2960             m_out.trap();
2961             m_out.unreachable();
2962             
2963             m_out.appendTo(continuation, lastNext);
2964             return;
2965         }
2966         
2967         if (verboseCompilationEnabled())
2968             dataLog("    OSR exit with value sources: ", m_valueSources, "\n");
2969         
2970         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.osrExit.size());
2971         unsigned index = m_ftlState.osrExit.size();
2972         
2973         m_ftlState.jitCode->osrExit.append(OSRExit(
2974             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
2975             m_codeOriginForExitTarget, m_codeOriginForExitProfile, m_lastSetOperand,
2976             m_valueSources.numberOfArguments(), m_valueSources.numberOfLocals()));
2977         m_ftlState.osrExit.append(OSRExitCompilationInfo());
2978         
2979         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
2980         OSRExitCompilationInfo& info = m_ftlState.osrExit.last();
2981
2982         LBasicBlock lastNext = 0;
2983         LBasicBlock continuation = 0;
2984         
2985         if (!Options::useLLVMOSRExitIntrinsic()) {
2986             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
2987             continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
2988             
2989             m_out.branch(failCondition, failCase, continuation);
2990
2991             m_out.appendTo(m_prologue);
2992             info.m_thunkAddress = buildAlloca(m_out.m_builder, m_out.intPtr);
2993         
2994             lastNext = m_out.appendTo(failCase, continuation);
2995         }
2996         
2997         if (Options::ftlOSRExitOmitsMarshalling()) {
2998             m_out.call(
2999                 m_out.intToPtr(
3000                     m_out.get(info.m_thunkAddress),
3001                     pointerType(functionType(m_out.voidType))));
3002         } else
3003             emitOSRExitCall(failCondition, index, exit, info, lowValue, direction, recovery);
3004         
3005         if (!Options::useLLVMOSRExitIntrinsic()) {
3006             m_out.unreachable();
3007             
3008             m_out.appendTo(continuation, lastNext);
3009         
3010             m_exitThunkGenerator.emitThunk(index);
3011         }
3012     }
3013     
3014     void emitOSRExitCall(
3015         LValue failCondition, unsigned index, OSRExit& exit, OSRExitCompilationInfo& info,
3016         FormattedValue lowValue, SpeculationDirection direction, FormattedValue recovery)
3017     {
3018         ExitArgumentList arguments;
3019         
3020         if (Options::useLLVMOSRExitIntrinsic()) {
3021             arguments.append(failCondition);
3022             arguments.append(m_out.constInt32(index));
3023         }
3024         
3025         arguments.append(m_callFrame);
3026         if (!!lowValue)
3027             arguments.append(lowValue.value());
3028         
3029         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
3030             ValueSource source = m_valueSources[i];
3031             
3032             switch (source.kind()) {
3033             case ValueInJSStack:
3034                 exit.m_values[i] = ExitValue::inJSStack();
3035                 break;
3036             case Int32InJSStack:
3037                 exit.m_values[i] = ExitValue::inJSStackAsInt32();
3038                 break;
3039             case DoubleInJSStack:
3040                 exit.m_values[i] = ExitValue::inJSStackAsDouble();
3041                 break;
3042             case SourceIsDead:
3043                 exit.m_values[i] = ExitValue::dead();
3044                 break;
3045             case HaveNode:
3046                 addExitArgumentForNode(exit, arguments, i, source.node());
3047                 break;
3048             default:
3049                 RELEASE_ASSERT_NOT_REACHED();
3050                 break;
3051             }
3052         }
3053         
3054         if (verboseCompilationEnabled())
3055             dataLog("        Exit values: ", exit.m_values, "\n");
3056         
3057         if (direction == ForwardSpeculation) {
3058             ASSERT(m_node);
3059             exit.convertToForward(m_highBlock, m_node, m_nodeIndex, recovery, arguments);
3060         }
3061         
3062         // So, the really lame thing here is that we have to build an LLVM function type.
3063         // Booo.
3064         Vector<LType, 16> argumentTypes;
3065         for (unsigned i = 0; i < arguments.size(); ++i)
3066             argumentTypes.append(typeOf(arguments[i]));
3067         
3068         if (Options::useLLVMOSRExitIntrinsic()) {
3069             m_out.call(m_out.osrExitIntrinsic(), arguments);
3070             return;
3071         }
3072         
3073         m_out.call(
3074             m_out.intToPtr(
3075                 m_out.get(info.m_thunkAddress),
3076                 pointerType(functionType(m_out.voidType, argumentTypes))),
3077             arguments);
3078     }
3079     
3080     void addExitArgumentForNode(
3081         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
3082     {
3083         ASSERT(node->shouldGenerate());
3084         ASSERT(node->hasResult());
3085
3086         if (tryToSetConstantExitArgument(exit, index, node))
3087             return;
3088         
3089         if (!isLive(node)) {
3090             bool found = false;
3091             
3092             if (permitsOSRBackwardRewiring(node->op())) {
3093                 node = node->child1().node();
3094                 if (tryToSetConstantExitArgument(exit, index, node))
3095                     return;
3096                 if (isLive(node))
3097                     found = true;
3098             }
3099             
3100             if (!found) {
3101                 Node* bestNode = 0;
3102                 unsigned bestScore = 0;
3103                 
3104                 HashSet<Node*>::iterator iter = m_live.begin();
3105                 HashSet<Node*>::iterator end = m_live.end();
3106                 for (; iter != end; ++iter) {
3107                     Node* candidate = *iter;
3108                     if (candidate->flags() & NodeHasVarArgs)
3109                         continue;
3110                     if (!candidate->child1())
3111                         continue;
3112                     if (candidate->child1() != node)
3113                         continue;
3114                     unsigned myScore = forwardRewiringSelectionScore(candidate->op());
3115                     if (myScore <= bestScore)
3116                         continue;
3117                     bestNode = candidate;
3118                     bestScore = myScore;
3119                 }
3120                 
3121                 if (bestNode) {
3122                     ASSERT(isLive(bestNode));
3123                     node = bestNode;
3124                     found = true;
3125                 }
3126             }
3127             
3128             if (!found) {
3129                 exit.m_values[index] = ExitValue::dead();