Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "FTLAbstractHeapRepository.h"
  35 #include "FTLExitThunkGenerator.h"
  36 #include "FTLForOSREntryJITCode.h"
  37 #include "FTLFormattedValue.h"
  38 #include "FTLLoweredNodeValue.h"
  39 #include "FTLOutput.h"
  40 #include "FTLThunks.h"
  41 #include "FTLValueSource.h"
  42 #include "LinkBuffer.h"
  43 #include "OperandsInlines.h"
  44 #include "Operations.h"
  45 #include "VirtualRegister.h"
  46
  47 #include <wtf/ProcessID.h>
  48
  49 namespace JSC { namespace FTL {
  50
  51 using namespace DFG;
  52
  53 static int compileCounter;
  54
  55 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  56 // significantly less dead code.
  57 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  58         FormattedValue _ftc_lowValue = (lowValue);                      \
  59         Edge _ftc_highValue = (highValue);                              \
  60         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  61         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  62             break;                                                      \
  63         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  64     } while (false)
  65
  66 class LowerDFGToLLVM {
  67 public:
  68     LowerDFGToLLVM(State& state)
  69         : m_graph(state.graph)
  70         , m_ftlState(state)
  71         , m_heaps(state.context)
  72         , m_out(state.context)
  73         , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
  74         , m_lastSetOperand(VirtualRegister())
  75         , m_exitThunkGenerator(state)
  76         , m_state(state.graph)
  77         , m_interpreter(state.graph, m_state)
  78     {
  79     }
  80
  81     void lower()
  82     {
  83         CString name;
  84         if (verboseCompilationEnabled()) {
  85             name = toCString(
  86                 "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
  87                 "_", codeBlock()->hash());
  88         } else
  89             name = "jsBody";
  90
  91         m_graph.m_dominators.computeIfNecessary(m_graph);
  92
  93         m_ftlState.module =
  94             LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
  95
  96         m_ftlState.function = addFunction(
  97             m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
  98         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
  99
 100         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 101
 102         m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
 103         m_out.appendTo(m_prologue);
 104         createPhiVariables();
 105
 106         m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
 107
 108         m_callFrame = m_out.param(0);
 109         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 110         m_tagMask = m_out.constInt64(TagMask);
 111
 112         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 113             m_highBlock = m_graph.block(blockIndex);
 114             if (!m_highBlock)
 115                 continue;
 116             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 117         }
 118
 119         Vector<BasicBlock*> depthFirst;
 120         m_graph.getBlocksInDepthFirstOrder(depthFirst);
 121         for (unsigned i = 0; i < depthFirst.size(); ++i)
 122             compileBlock(depthFirst[i]);
 123
 124         // And now complete the initialization block.
 125         linkOSRExitsAndCompleteInitializationBlocks();
 126
 127         if (Options::dumpLLVMIR())
 128             dumpModule(m_ftlState.module);
 129
 130         if (verboseCompilationEnabled())
 131             m_ftlState.dumpState("after lowering");
 132         if (validationEnabled())
 133             verifyModule(m_ftlState.module);
 134     }
 135
 136 private:
 137
 138     void createPhiVariables()
 139     {
 140         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 141             BasicBlock* block = m_graph.block(blockIndex);
 142             if (!block)
 143                 continue;
 144             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 145                 Node* node = block->at(nodeIndex);
 146                 if (node->op() != Phi)
 147                     continue;
 148                 LType type;
 149                 switch (node->flags() & NodeResultMask) {
 150                 case NodeResultNumber:
 151                     type = m_out.doubleType;
 152                     break;
 153                 case NodeResultInt32:
 154                     type = m_out.int32;
 155                     break;
 156                 case NodeResultInt52:
 157                     type = m_out.int64;
 158                     break;
 159                 case NodeResultBoolean:
 160                     type = m_out.boolean;
 161                     break;
 162                 case NodeResultJS:
 163                     type = m_out.int64;
 164                     break;
 165                 default:
 166                     RELEASE_ASSERT_NOT_REACHED();
 167                     break;
 168                 }
 169                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 170             }
 171         }
 172     }
 173
 174     void compileBlock(BasicBlock* block)
 175     {
 176         if (!block)
 177             return;
 178
 179         if (verboseCompilationEnabled())
 180             dataLog("Compiling block ", *block, "\n");
 181
 182         m_highBlock = block;
 183
 184         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 185
 186         m_nextHighBlock = 0;
 187         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 188             m_nextHighBlock = m_graph.block(nextBlockIndex);
 189             if (m_nextHighBlock)
 190                 break;
 191         }
 192         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 193
 194         // All of this effort to find the next block gives us the ability to keep the
 195         // generated IR in roughly program order. This ought not affect the performance
 196         // of the generated code (since we expect LLVM to reorder things) but it will
 197         // make IR dumps easier to read.
 198         m_out.appendTo(lowBlock, m_nextLowBlock);
 199
 200         if (Options::ftlCrashes())
 201             m_out.crashNonTerminal();
 202
 203         if (!m_highBlock->cfaHasVisited) {
 204             m_out.crash();
 205             return;
 206         }
 207
 208         initializeOSRExitStateForBlock();
 209
 210         m_live = block->ssa->liveAtHead;
 211
 212         m_state.reset();
 213         m_state.beginBasicBlock(m_highBlock);
 214
 215         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 216             if (!compileNode(m_nodeIndex))
 217                 break;
 218         }
 219     }
 220
 221     bool compileNode(unsigned nodeIndex)
 222     {
 223         if (!m_state.isValid()) {
 224             m_out.unreachable();
 225             return false;
 226         }
 227
 228         m_node = m_highBlock->at(nodeIndex);
 229         m_codeOriginForExitProfile = m_node->codeOrigin;
 230         m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
 231
 232         if (verboseCompilationEnabled())
 233             dataLog("Lowering ", m_node, "\n");
 234
 235         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
 236
 237         m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
 238
 239         switch (m_node->op()) {
 240         case Upsilon:
 241             compileUpsilon();
 242             break;
 243         case Phi:
 244             compilePhi();
 245             break;
 246         case JSConstant:
 247             break;
 248         case WeakJSConstant:
 249             compileWeakJSConstant();
 250             break;
 251         case GetArgument:
 252             compileGetArgument();
 253             break;
 254         case ExtractOSREntryLocal:
 255             compileExtractOSREntryLocal();
 256             break;
 257         case GetLocal:
 258             compileGetLocal();
 259             break;
 260         case SetLocal:
 261             compileSetLocal();
 262             break;
 263         case MovHint:
 264             compileMovHint();
 265             break;
 266         case ZombieHint:
 267             compileZombieHint();
 268             break;
 269         case MovHintAndCheck:
 270             compileMovHintAndCheck();
 271             break;
 272         case Phantom:
 273             compilePhantom();
 274             break;
 275         case Flush:
 276         case PhantomLocal:
 277         case SetArgument:
 278         case LoopHint:
 279             break;
 280         case ArithAdd:
 281         case ValueAdd:
 282             compileAdd();
 283             break;
 284         case ArithSub:
 285             compileArithSub();
 286             break;
 287         case ArithMul:
 288             compileArithMul();
 289             break;
 290         case ArithDiv:
 291             compileArithDiv();
 292             break;
 293         case ArithMod:
 294             compileArithMod();
 295             break;
 296         case ArithMin:
 297         case ArithMax:
 298             compileArithMinOrMax();
 299             break;
 300         case ArithAbs:
 301             compileArithAbs();
 302             break;
 303         case ArithNegate:
 304             compileArithNegate();
 305             break;
 306         case BitAnd:
 307             compileBitAnd();
 308             break;
 309         case BitOr:
 310             compileBitOr();
 311             break;
 312         case BitXor:
 313             compileBitXor();
 314             break;
 315         case BitRShift:
 316             compileBitRShift();
 317             break;
 318         case BitLShift:
 319             compileBitLShift();
 320             break;
 321         case BitURShift:
 322             compileBitURShift();
 323             break;
 324         case UInt32ToNumber:
 325             compileUInt32ToNumber();
 326             break;
 327         case Int32ToDouble:
 328             compileInt32ToDouble();
 329             break;
 330         case CheckStructure:
 331             compileCheckStructure();
 332             break;
 333         case StructureTransitionWatchpoint:
 334             compileStructureTransitionWatchpoint();
 335             break;
 336         case ArrayifyToStructure:
 337             compileArrayifyToStructure();
 338             break;
 339         case PutStructure:
 340             compilePutStructure();
 341             break;
 342         case PhantomPutStructure:
 343             compilePhantomPutStructure();
 344             break;
 345         case GetButterfly:
 346             compileGetButterfly();
 347             break;
 348         case GetIndexedPropertyStorage:
 349             compileGetIndexedPropertyStorage();
 350             break;
 351         case CheckArray:
 352             compileCheckArray();
 353             break;
 354         case GetArrayLength:
 355             compileGetArrayLength();
 356             break;
 357         case GetByVal:
 358             compileGetByVal();
 359             break;
 360         case PutByVal:
 361         case PutByValAlias:
 362             compilePutByVal();
 363             break;
 364         case GetByOffset:
 365             compileGetByOffset();
 366             break;
 367         case PutByOffset:
 368             compilePutByOffset();
 369             break;
 370         case GetGlobalVar:
 371             compileGetGlobalVar();
 372             break;
 373         case PutGlobalVar:
 374             compilePutGlobalVar();
 375             break;
 376         case GlobalVarWatchpoint:
 377             compileGlobalVarWatchpoint();
 378             break;
 379         case GetMyScope:
 380             compileGetMyScope();
 381             break;
 382         case SkipScope:
 383             compileSkipScope();
 384             break;
 385         case GetClosureRegisters:
 386             compileGetClosureRegisters();
 387             break;
 388         case GetClosureVar:
 389             compileGetClosureVar();
 390             break;
 391         case PutClosureVar:
 392             compilePutClosureVar();
 393             break;
 394         case CompareEq:
 395             compileCompareEq();
 396             break;
 397         case CompareEqConstant:
 398             compileCompareEqConstant();
 399             break;
 400         case CompareStrictEq:
 401             compileCompareStrictEq();
 402             break;
 403         case CompareStrictEqConstant:
 404             compileCompareStrictEqConstant();
 405             break;
 406         case CompareLess:
 407             compileCompareLess();
 408             break;
 409         case CompareLessEq:
 410             compileCompareLessEq();
 411             break;
 412         case CompareGreater:
 413             compileCompareGreater();
 414             break;
 415         case CompareGreaterEq:
 416             compileCompareGreaterEq();
 417             break;
 418         case LogicalNot:
 419             compileLogicalNot();
 420             break;
 421         case Call:
 422         case Construct:
 423             compileCallOrConstruct();
 424             break;
 425         case Jump:
 426             compileJump();
 427             break;
 428         case Branch:
 429             compileBranch();
 430             break;
 431         case Switch:
 432             compileSwitch();
 433             break;
 434         case Return:
 435             compileReturn();
 436             break;
 437         case ForceOSRExit:
 438             compileForceOSRExit();
 439             break;
 440         default:
 441             RELEASE_ASSERT_NOT_REACHED();
 442             break;
 443         }
 444
 445         if (m_node->shouldGenerate())
 446             DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
 447
 448         if (m_node->adjustedRefCount())
 449             m_live.add(m_node);
 450
 451         if (shouldExecuteEffects)
 452             m_interpreter.executeEffects(nodeIndex);
 453
 454         return true;
 455     }
 456
 457     void compileUpsilon()
 458     {
 459         LValue destination = m_phis.get(m_node->phi());
 460
 461         switch (m_node->child1().useKind()) {
 462         case NumberUse:
 463             m_out.set(lowDouble(m_node->child1()), destination);
 464             break;
 465         case Int32Use:
 466             m_out.set(lowInt32(m_node->child1()), destination);
 467             break;
 468         case MachineIntUse:
 469             m_out.set(lowInt52(m_node->child1()), destination);
 470             break;
 471         case BooleanUse:
 472             m_out.set(lowBoolean(m_node->child1()), destination);
 473             break;
 474         case CellUse:
 475             m_out.set(lowCell(m_node->child1()), destination);
 476             break;
 477         case UntypedUse:
 478             m_out.set(lowJSValue(m_node->child1()), destination);
 479             break;
 480         default:
 481             RELEASE_ASSERT_NOT_REACHED();
 482             break;
 483         }
 484     }
 485
 486     void compilePhi()
 487     {
 488         LValue source = m_phis.get(m_node);
 489
 490         switch (m_node->flags() & NodeResultMask) {
 491         case NodeResultNumber:
 492             setDouble(m_out.get(source));
 493             break;
 494         case NodeResultInt32:
 495             setInt32(m_out.get(source));
 496             break;
 497         case NodeResultInt52:
 498             setInt52(m_out.get(source));
 499             break;
 500         case NodeResultBoolean:
 501             setBoolean(m_out.get(source));
 502             break;
 503         case NodeResultJS:
 504             setJSValue(m_out.get(source));
 505             break;
 506         default:
 507             RELEASE_ASSERT_NOT_REACHED();
 508             break;
 509         }
 510     }
 511
 512     void compileJSConstant()
 513     {
 514         JSValue value = m_graph.valueOfJSConstant(m_node);
 515         if (value.isDouble())
 516             setDouble(m_out.constDouble(value.asDouble()));
 517         else
 518             setJSValue(m_out.constInt64(JSValue::encode(value)));
 519     }
 520
 521     void compileWeakJSConstant()
 522     {
 523         setJSValue(weakPointer(m_node->weakConstant()));
 524     }
 525
 526     void compileGetArgument()
 527     {
 528         VariableAccessData* variable = m_node->variableAccessData();
 529         VirtualRegister operand = variable->local();
 530
 531         LValue jsValue = m_out.load64(addressFor(operand));
 532
 533         switch (useKindFor(variable->flushFormat())) {
 534         case Int32Use:
 535             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
 536             setInt32(unboxInt32(jsValue));
 537             break;
 538         case CellUse:
 539             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
 540             setJSValue(jsValue);
 541             break;
 542         case BooleanUse:
 543             speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
 544             setBoolean(unboxBoolean(jsValue));
 545             break;
 546         case UntypedUse:
 547             setJSValue(jsValue);
 548             break;
 549         default:
 550             RELEASE_ASSERT_NOT_REACHED();
 551             break;
 552         }
 553     }
 554
 555     void compileExtractOSREntryLocal()
 556     {
 557         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
 558             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
 559         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
 560     }
 561
 562     void compileGetLocal()
 563     {
 564         // GetLocals arise only for captured variables.
 565
 566         VariableAccessData* variable = m_node->variableAccessData();
 567         AbstractValue& value = m_state.variables().operand(variable->local());
 568
 569         RELEASE_ASSERT(variable->isCaptured());
 570
 571         if (isInt32Speculation(value.m_type))
 572             setInt32(m_out.load32(payloadFor(variable->local())));
 573         else
 574             setJSValue(m_out.load64(addressFor(variable->local())));
 575     }
 576
 577     void compileSetLocal()
 578     {
 579         observeMovHint(m_node);
 580
 581         VariableAccessData* variable = m_node->variableAccessData();
 582         switch (variable->flushFormat()) {
 583         case FlushedJSValue: {
 584             LValue value = lowJSValue(m_node->child1());
 585             m_out.store64(value, addressFor(variable->local()));
 586             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
 587             return;
 588         }
 589
 590         case FlushedDouble: {
 591             LValue value = lowDouble(m_node->child1());
 592             m_out.storeDouble(value, addressFor(variable->local()));
 593             m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
 594             return;
 595         }
 596
 597         case FlushedInt32: {
 598             LValue value = lowInt32(m_node->child1());
 599             m_out.store32(value, payloadFor(variable->local()));
 600             m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
 601             return;
 602         }
 603
 604         case FlushedInt52: {
 605             LValue value = lowInt52(m_node->child1());
 606             m_out.store64(value, addressFor(variable->local()));
 607             m_valueSources.operand(variable->local()) = ValueSource(Int52InJSStack);
 608             return;
 609         }
 610
 611         case FlushedCell: {
 612             LValue value = lowCell(m_node->child1());
 613             m_out.store64(value, addressFor(variable->local()));
 614             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
 615             return;
 616         }
 617
 618         case FlushedBoolean: {
 619             speculateBoolean(m_node->child1());
 620             m_out.store64(
 621                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
 622                 addressFor(variable->local()));
 623             m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
 624             return;
 625         }
 626
 627         case DeadFlush:
 628             RELEASE_ASSERT_NOT_REACHED();
 629         }
 630
 631         RELEASE_ASSERT_NOT_REACHED();
 632     }
 633
 634     void compileMovHint()
 635     {
 636         observeMovHint(m_node);
 637     }
 638
 639     void compileZombieHint()
 640     {
 641         VariableAccessData* data = m_node->variableAccessData();
 642         m_lastSetOperand = data->local();
 643         m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
 644     }
 645
 646     void compileMovHintAndCheck()
 647     {
 648         observeMovHint(m_node);
 649         speculate(m_node->child1());
 650     }
 651
 652     void compilePhantom()
 653     {
 654         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
 655     }
 656
 657     void compileAdd()
 658     {
 659         switch (m_node->binaryUseKind()) {
 660         case Int32Use: {
 661             LValue left = lowInt32(m_node->child1());
 662             LValue right = lowInt32(m_node->child2());
 663
 664             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags())) {
 665                 setInt32(m_out.add(left, right));
 666                 break;
 667             }
 668
 669             LValue result = m_out.addWithOverflow32(left, right);
 670             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
 671             setInt32(m_out.extractValue(result, 0));
 672             break;
 673         }
 674
 675         case MachineIntUse: {
 676             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)
 677                 && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) {
 678                 Int52Kind kind;
 679                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
 680                 LValue right = lowInt52(m_node->child2(), kind);
 681                 setInt52(m_out.add(left, right), kind);
 682                 break;
 683             }
 684
 685             LValue left = lowInt52(m_node->child1());
 686             LValue right = lowInt52(m_node->child2());
 687             LValue result = m_out.addWithOverflow64(left, right);
 688             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
 689             setInt52(m_out.extractValue(result, 0));
 690             break;
 691         }
 692
 693         case NumberUse: {
 694             setDouble(
 695                 m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
 696             break;
 697         }
 698
 699         default:
 700             RELEASE_ASSERT_NOT_REACHED();
 701             break;
 702         }
 703     }
 704
 705     void compileArithSub()
 706     {
 707         switch (m_node->binaryUseKind()) {
 708         case Int32Use: {
 709             LValue left = lowInt32(m_node->child1());
 710             LValue right = lowInt32(m_node->child2());
 711
 712             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags())) {
 713                 setInt32(m_out.sub(left, right));
 714                 break;
 715             }
 716
 717             LValue result = m_out.subWithOverflow32(left, right);
 718             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
 719             setInt32(m_out.extractValue(result, 0));
 720             break;
 721         }
 722
 723         case MachineIntUse: {
 724             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)
 725                 && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) {
 726                 Int52Kind kind;
 727                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
 728                 LValue right = lowInt52(m_node->child2(), kind);
 729                 setInt52(m_out.sub(left, right), kind);
 730                 break;
 731             }
 732
 733             LValue left = lowInt52(m_node->child1());
 734             LValue right = lowInt52(m_node->child2());
 735             LValue result = m_out.subWithOverflow64(left, right);
 736             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
 737             setInt52(m_out.extractValue(result, 0));
 738             break;
 739         }
 740
 741         case NumberUse: {
 742             setDouble(
 743                 m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
 744             break;
 745         }
 746
 747         default:
 748             RELEASE_ASSERT_NOT_REACHED();
 749             break;
 750         }
 751     }
 752
 753     void compileArithMul()
 754     {
 755         switch (m_node->binaryUseKind()) {
 756         case Int32Use: {
 757             LValue left = lowInt32(m_node->child1());
 758             LValue right = lowInt32(m_node->child2());
 759
 760             LValue result;
 761             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags()))
 762                 result = m_out.mul(left, right);
 763             else {
 764                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
 765                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
 766                 result = m_out.extractValue(overflowResult, 0);
 767             }
 768
 769             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
 770                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
 771                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
 772
 773                 m_out.branch(m_out.notZero32(result), continuation, slowCase);
 774
 775                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
 776                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
 777                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
 778                 m_out.jump(continuation);
 779                 m_out.appendTo(continuation, lastNext);
 780             }
 781
 782             setInt32(result);
 783             break;
 784         }
 785
 786         case MachineIntUse: {
 787             Int52Kind kind;
 788             LValue left = lowWhicheverInt52(m_node->child1(), kind);
 789             LValue right = lowInt52(m_node->child2(), opposite(kind));
 790
 791             LValue overflowResult = m_out.mulWithOverflow64(left, right);
 792             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
 793             LValue result = m_out.extractValue(overflowResult, 0);
 794
 795             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
 796                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
 797                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
 798
 799                 m_out.branch(m_out.notZero64(result), continuation, slowCase);
 800
 801                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
 802                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int64Zero));
 803                 speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int64Zero));
 804                 m_out.jump(continuation);
 805                 m_out.appendTo(continuation, lastNext);
 806             }
 807
 808             setInt52(result);
 809             break;
 810         }
 811
 812         case NumberUse: {
 813             setDouble(
 814                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
 815             break;
 816         }
 817
 818         default:
 819             RELEASE_ASSERT_NOT_REACHED();
 820             break;
 821         }
 822     }
 823
 824     void compileArithDiv()
 825     {
 826         switch (m_node->binaryUseKind()) {
 827         case Int32Use: {
 828             LValue numerator = lowInt32(m_node->child1());
 829             LValue denominator = lowInt32(m_node->child2());
 830
 831             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
 832             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
 833             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
 834
 835             Vector<ValueFromBlock, 3> results;
 836
 837             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
 838
 839             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
 840
 841             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
 842
 843             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
 844
 845             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
 846                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
 847                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
 848                 m_out.jump(continuation);
 849             } else {
 850                 // This is the case where we convert the result to an int after we're done. So,
 851                 // if the denominator is zero, then the result should be result should be zero.
 852                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
 853                 // check above) and the numerator is -2^31 then the result should be -2^31.
 854
 855                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
 856                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
 857                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
 858
 859                 m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
 860
 861                 m_out.appendTo(divByZero, notDivByZero);
 862                 results.append(m_out.anchor(m_out.int32Zero));
 863                 m_out.jump(done);
 864
 865                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
 866                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
 867
 868                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
 869                 results.append(m_out.anchor(neg2ToThe31));
 870                 m_out.jump(done);
 871             }
 872
 873             m_out.appendTo(continuation, done);
 874
 875             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
 876                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
 877                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
 878
 879                 m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
 880
 881                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
 882
 883                 speculate(
 884                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
 885
 886                 m_out.jump(numeratorContinuation);
 887
 888                 m_out.appendTo(numeratorContinuation, innerLastNext);
 889             }
 890
 891             LValue divisionResult = m_out.div(numerator, denominator);
 892
 893             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
 894                 speculate(
 895                     Overflow, noValue(), 0,
 896                     m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
 897             }
 898
 899             results.append(m_out.anchor(divisionResult));
 900             m_out.jump(done);
 901
 902             m_out.appendTo(done, lastNext);
 903
 904             setInt32(m_out.phi(m_out.int32, results));
 905             break;
 906         }
 907
 908         case NumberUse: {
 909             setDouble(
 910                 m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
 911             break;
 912         }
 913
 914         default:
 915             RELEASE_ASSERT_NOT_REACHED();
 916             break;
 917         }
 918     }
 919
 920     void compileArithMod()
 921     {
 922         switch (m_node->binaryUseKind()) {
 923         case Int32Use: {
 924             LValue numerator = lowInt32(m_node->child1());
 925             LValue denominator = lowInt32(m_node->child2());
 926
 927             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
 928             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
 929             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
 930
 931             Vector<ValueFromBlock, 3> results;
 932
 933             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
 934
 935             m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
 936
 937             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
 938
 939             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
 940
 941             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
 942             // separate case for that. But it probably doesn't matter so much.
 943             if (bytecodeUsesAsNumber(m_node->arithNodeFlags())) {
 944                 speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
 945                 speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
 946                 m_out.jump(continuation);
 947             } else {
 948                 // This is the case where we convert the result to an int after we're done. So,
 949                 // if the denominator is zero, then the result should be result should be zero.
 950                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
 951                 // check above) and the numerator is -2^31 then the result should be -2^31.
 952
 953                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
 954                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
 955                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
 956
 957                 m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
 958
 959                 m_out.appendTo(modByZero, notModByZero);
 960                 results.append(m_out.anchor(m_out.int32Zero));
 961                 m_out.jump(done);
 962
 963                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
 964                 m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
 965
 966                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
 967                 results.append(m_out.anchor(m_out.int32Zero));
 968                 m_out.jump(done);
 969             }
 970
 971             m_out.appendTo(continuation, done);
 972
 973             LValue remainder = m_out.rem(numerator, denominator);
 974
 975             if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
 976                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
 977                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
 978
 979                 m_out.branch(
 980                     m_out.lessThan(numerator, m_out.int32Zero),
 981                     negativeNumerator, numeratorContinuation);
 982
 983                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
 984
 985                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
 986
 987                 m_out.jump(numeratorContinuation);
 988
 989                 m_out.appendTo(numeratorContinuation, innerLastNext);
 990             }
 991
 992             results.append(m_out.anchor(remainder));
 993             m_out.jump(done);
 994
 995             m_out.appendTo(done, lastNext);
 996
 997             setInt32(m_out.phi(m_out.int32, results));
 998             break;
 999         }
1000
1001         case NumberUse: {
1002             setDouble(
1003                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1004             break;
1005         }
1006
1007         default:
1008             RELEASE_ASSERT_NOT_REACHED();
1009             break;
1010         }
1011     }
1012
1013     void compileArithMinOrMax()
1014     {
1015         switch (m_node->binaryUseKind()) {
1016         case Int32Use: {
1017             LValue left = lowInt32(m_node->child1());
1018             LValue right = lowInt32(m_node->child2());
1019
1020             setInt32(
1021                 m_out.select(
1022                     m_node->op() == ArithMin
1023                         ? m_out.lessThan(left, right)
1024                         : m_out.lessThan(right, left),
1025                     left, right));
1026             break;
1027         }
1028
1029         case NumberUse: {
1030             LValue left = lowDouble(m_node->child1());
1031             LValue right = lowDouble(m_node->child2());
1032
1033             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1034             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1035
1036             Vector<ValueFromBlock, 2> results;
1037
1038             results.append(m_out.anchor(left));
1039             m_out.branch(
1040                 m_node->op() == ArithMin
1041                     ? m_out.doubleLessThan(left, right)
1042                     : m_out.doubleGreaterThan(left, right),
1043                 continuation, notLessThan);
1044
1045             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1046             results.append(m_out.anchor(m_out.select(
1047                 m_node->op() == ArithMin
1048                     ? m_out.doubleGreaterThanOrEqual(left, right)
1049                     : m_out.doubleLessThanOrEqual(left, right),
1050                 right, m_out.constDouble(0.0 / 0.0))));
1051             m_out.jump(continuation);
1052
1053             m_out.appendTo(continuation, lastNext);
1054             setDouble(m_out.phi(m_out.doubleType, results));
1055             break;
1056         }
1057
1058         default:
1059             RELEASE_ASSERT_NOT_REACHED();
1060             break;
1061         }
1062     }
1063
1064     void compileArithAbs()
1065     {
1066         switch (m_node->child1().useKind()) {
1067         case Int32Use: {
1068             LValue value = lowInt32(m_node->child1());
1069
1070             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1071             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1072
1073             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1074
1075             setInt32(result);
1076             break;
1077         }
1078
1079         case NumberUse: {
1080             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1081             break;
1082         }
1083
1084         default:
1085             RELEASE_ASSERT_NOT_REACHED();
1086             break;
1087         }
1088     }
1089
1090     void compileArithNegate()
1091     {
1092         switch (m_node->child1().useKind()) {
1093         case Int32Use: {
1094             LValue value = lowInt32(m_node->child1());
1095
1096             LValue result;
1097             if (bytecodeCanTruncateInteger(m_node->arithNodeFlags()))
1098                 result = m_out.neg(value);
1099             else if (bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
1100                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1101                 // does the trick, though.
1102                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1103                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1104                 result = m_out.extractValue(overflowResult, 0);
1105             } else {
1106                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1107                 result = m_out.neg(value);
1108             }
1109
1110             setInt32(result);
1111             break;
1112         }
1113
1114         case MachineIntUse: {
1115             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)) {
1116                 Int52Kind kind;
1117                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1118                 LValue result = m_out.neg(value);
1119                 if (!bytecodeCanIgnoreNegativeZero(m_node->arithNodeFlags()))
1120                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1121                 setInt52(result, kind);
1122                 break;
1123             }
1124
1125             LValue value = lowInt52(m_node->child1());
1126             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1127             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1128             LValue result = m_out.extractValue(overflowResult, 0);
1129             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1130             setInt52(result);
1131             break;
1132         }
1133
1134         case NumberUse: {
1135             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1136             break;
1137         }
1138
1139         default:
1140             RELEASE_ASSERT_NOT_REACHED();
1141             break;
1142         }
1143     }
1144
1145     void compileBitAnd()
1146     {
1147         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1148     }
1149
1150     void compileBitOr()
1151     {
1152         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1153     }
1154
1155     void compileBitXor()
1156     {
1157         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1158     }
1159
1160     void compileBitRShift()
1161     {
1162         setInt32(m_out.aShr(
1163             lowInt32(m_node->child1()),
1164             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1165     }
1166
1167     void compileBitLShift()
1168     {
1169         setInt32(m_out.shl(
1170             lowInt32(m_node->child1()),
1171             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1172     }
1173
1174     void compileBitURShift()
1175     {
1176         setInt32(m_out.lShr(
1177             lowInt32(m_node->child1()),
1178             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1179     }
1180
1181     void compileUInt32ToNumber()
1182     {
1183         LValue value = lowInt32(m_node->child1());
1184
1185         if (!nodeCanSpeculateInt32(m_node->arithNodeFlags())) {
1186             setDouble(m_out.unsignedToDouble(value));
1187             return;
1188         }
1189
1190         speculateForward(
1191             Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
1192             FormattedValue(ValueFormatUInt32, value));
1193         setInt32(value);
1194     }
1195
1196     void compileInt32ToDouble()
1197     {
1198         // This node is tricky to compile in the DFG backend because it tries to
1199         // avoid converting child1 to a double in-place, as that would make subsequent
1200         // int uses of of child1 fail. But the FTL needs no such special magic, since
1201         // unlike the DFG backend, the FTL allows each node to have multiple
1202         // contemporaneous low-level representations. So, this gives child1 a double
1203         // representation and then forwards that representation to m_node.
1204
1205         setDouble(lowDouble(m_node->child1()));
1206     }
1207
1208     void compileCheckStructure()
1209     {
1210         LValue cell = lowCell(m_node->child1());
1211
1212         ExitKind exitKind;
1213         if (m_node->child1()->op() == WeakJSConstant)
1214             exitKind = BadWeakConstantCache;
1215         else
1216             exitKind = BadCache;
1217
1218         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1219
1220         if (m_node->structureSet().size() == 1) {
1221             speculate(
1222                 exitKind, jsValueValue(cell), 0,
1223                 m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
1224             return;
1225         }
1226
1227         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1228
1229         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1230         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1231             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1232             m_out.branch(
1233                 m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
1234                 continuation, nextStructure);
1235             m_out.appendTo(nextStructure);
1236         }
1237
1238         speculate(
1239             exitKind, jsValueValue(cell), 0,
1240             m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
1241
1242         m_out.jump(continuation);
1243         m_out.appendTo(continuation, lastNext);
1244     }
1245
1246     void compileStructureTransitionWatchpoint()
1247     {
1248         addWeakReference(m_node->structure());
1249
1250         // FIXME: Implement structure transition watchpoints.
1251         // https://bugs.webkit.org/show_bug.cgi?id=113647
1252
1253         speculateCell(m_node->child1());
1254     }
1255
1256     void compileArrayifyToStructure()
1257     {
1258         LValue cell = lowCell(m_node->child1());
1259         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1260
1261         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1262         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1263
1264         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1265
1266         m_out.branch(
1267             m_out.notEqual(structure, weakPointer(m_node->structure())),
1268             unexpectedStructure, continuation);
1269
1270         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1271
1272         if (property) {
1273             switch (m_node->arrayMode().type()) {
1274             case Array::Int32:
1275             case Array::Double:
1276             case Array::Contiguous:
1277                 speculate(
1278                     Uncountable, noValue(), 0,
1279                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1280                 break;
1281             default:
1282                 break;
1283             }
1284         }
1285
1286         switch (m_node->arrayMode().type()) {
1287         case Array::Int32:
1288             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1289             break;
1290         case Array::Double:
1291             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1292             break;
1293         case Array::Contiguous:
1294             if (m_node->arrayMode().conversion() == Array::RageConvert)
1295                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1296             else
1297                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1298             break;
1299         case Array::ArrayStorage:
1300         case Array::SlowPutArrayStorage:
1301             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1302             break;
1303         default:
1304             RELEASE_ASSERT_NOT_REACHED();
1305             break;
1306         }
1307
1308         structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
1309         speculate(
1310             BadIndexingType, jsValueValue(cell), 0,
1311             m_out.notEqual(structure, weakPointer(m_node->structure())));
1312         m_out.jump(continuation);
1313
1314         m_out.appendTo(continuation, lastNext);
1315     }
1316
1317     void compilePutStructure()
1318     {
1319         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1320
1321         m_out.store64(
1322             m_out.constIntPtr(m_node->structureTransitionData().newStructure),
1323             lowCell(m_node->child1()), m_heaps.JSCell_structure);
1324     }
1325
1326     void compilePhantomPutStructure()
1327     {
1328         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1329     }
1330
1331     void compileGetButterfly()
1332     {
1333         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1334     }
1335
1336     void compileGetIndexedPropertyStorage()
1337     {
1338         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_vector));
1339     }
1340
1341     void compileCheckArray()
1342     {
1343         Edge edge = m_node->child1();
1344         LValue cell = lowCell(edge);
1345
1346         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge)))
1347             return;
1348
1349         speculate(
1350             BadIndexingType, jsValueValue(cell), 0,
1351             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
1352     }
1353
1354     void compileGetArrayLength()
1355     {
1356         switch (m_node->arrayMode().type()) {
1357         case Array::Int32:
1358         case Array::Double:
1359         case Array::Contiguous: {
1360             setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1361             return;
1362         }
1363
1364         default:
1365             if (isTypedView(m_node->arrayMode().typedArrayType())) {
1366                 setInt32(
1367                     m_out.load32(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
1368                 return;
1369             }
1370
1371             RELEASE_ASSERT_NOT_REACHED();
1372             return;
1373         }
1374     }
1375
1376     void compileGetByVal()
1377     {
1378         LValue index = lowInt32(m_node->child2());
1379         LValue storage = lowStorage(m_node->child3());
1380
1381         switch (m_node->arrayMode().type()) {
1382         case Array::Int32:
1383         case Array::Contiguous: {
1384             if (m_node->arrayMode().isInBounds()) {
1385                 speculate(
1386                     OutOfBounds, noValue(), 0,
1387                     m_out.aboveOrEqual(
1388                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1389
1390                 LValue result = m_out.load64(m_out.baseIndex(
1391                     m_node->arrayMode().type() == Array::Int32 ?
1392                         m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1393                     storage, m_out.zeroExt(index, m_out.intPtr),
1394                     m_state.forNode(m_node->child2()).m_value));
1395                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1396                 setJSValue(result);
1397                 return;
1398             }
1399
1400             // FIXME: Implement hole/OOB loads in the FTL.
1401             // https://bugs.webkit.org/show_bug.cgi?id=118077
1402             RELEASE_ASSERT_NOT_REACHED();
1403             return;
1404         }
1405
1406         case Array::Double: {
1407             if (m_node->arrayMode().isInBounds()) {
1408                 if (m_node->arrayMode().isSaneChain()) {
1409                     // FIXME: Implement structure transition watchpoints.
1410                     // https://bugs.webkit.org/show_bug.cgi?id=113647
1411                 }
1412
1413                 speculate(
1414                     OutOfBounds, noValue(), 0,
1415                     m_out.aboveOrEqual(
1416                         index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
1417
1418                 LValue result = m_out.loadDouble(m_out.baseIndex(
1419                     m_heaps.indexedDoubleProperties,
1420                     storage, m_out.zeroExt(index, m_out.intPtr),
1421                     m_state.forNode(m_node->child2()).m_value));
1422
1423                 if (!m_node->arrayMode().isSaneChain()) {
1424                     speculate(
1425                         LoadFromHole, noValue(), 0,
1426                         m_out.doubleNotEqualOrUnordered(result, result));
1427                 }
1428                 setDouble(result);
1429                 break;
1430             }
1431
1432             // FIXME: Implement hole/OOB loads in the FTL.
1433             // https://bugs.webkit.org/show_bug.cgi?id=118077
1434             RELEASE_ASSERT_NOT_REACHED();
1435             return;
1436         }
1437
1438         default: {
1439             TypedArrayType type = m_node->arrayMode().typedArrayType();
1440
1441             if (isTypedView(type)) {
1442                 LValue array = lowCell(m_node->child1());
1443
1444                 speculate(
1445                     OutOfBounds, noValue(), 0,
1446                     m_out.aboveOrEqual(
1447                         index, m_out.load32(array, m_heaps.JSArrayBufferView_length)));
1448
1449                 TypedPointer pointer = TypedPointer(
1450                     m_heaps.typedArrayProperties,
1451                     m_out.add(
1452                         storage,
1453                         m_out.shl(
1454                             m_out.zeroExt(index, m_out.intPtr),
1455                             m_out.constIntPtr(logElementSize(type)))));
1456
1457                 if (isInt(type)) {
1458                     LValue result;
1459                     switch (elementSize(type)) {
1460                     case 1:
1461                         result = m_out.load8(pointer);
1462                         break;
1463                     case 2:
1464                         result = m_out.load16(pointer);
1465                         break;
1466                     case 4:
1467                         result = m_out.load32(pointer);
1468                         break;
1469                     default:
1470                         RELEASE_ASSERT_NOT_REACHED();
1471                     }
1472
1473                     if (elementSize(type) < 4) {
1474                         if (isSigned(type))
1475                             result = m_out.signExt(result, m_out.int32);
1476                         else
1477                             result = m_out.zeroExt(result, m_out.int32);
1478                         setInt32(result);
1479                         return;
1480                     }
1481
1482                     if (isSigned(type)) {
1483                         setInt32(result);
1484                         return;
1485                     }
1486
1487                     if (m_node->shouldSpeculateInt32()) {
1488                         speculateForward(
1489                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero),
1490                             uInt32Value(result));
1491                         setInt32(result);
1492                         return;
1493                     }
1494
1495                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
1496                     return;
1497                 }
1498
1499                 ASSERT(isFloat(type));
1500
1501                 LValue result;
1502                 switch (type) {
1503                 case TypeFloat32:
1504                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
1505                     break;
1506                 case TypeFloat64:
1507                     result = m_out.loadDouble(pointer);
1508                     break;
1509                 default:
1510                     RELEASE_ASSERT_NOT_REACHED();
1511                 }
1512
1513                 result = m_out.select(
1514                     m_out.doubleEqual(result, result), result, m_out.constDouble(QNaN));
1515                 setDouble(result);
1516                 return;
1517             }
1518
1519             RELEASE_ASSERT_NOT_REACHED();
1520             return;
1521         } }
1522     }
1523
1524     void compilePutByVal()
1525     {
1526         Edge child1 = m_graph.varArgChild(m_node, 0);
1527         Edge child2 = m_graph.varArgChild(m_node, 1);
1528         Edge child3 = m_graph.varArgChild(m_node, 2);
1529         Edge child4 = m_graph.varArgChild(m_node, 3);
1530
1531         LValue base = lowCell(child1);
1532         LValue index = lowInt32(child2);
1533         LValue storage = lowStorage(child4);
1534
1535         switch (m_node->arrayMode().type()) {
1536         case Array::Int32:
1537         case Array::Double:
1538         case Array::Contiguous: {
1539             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
1540             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
1541
1542             switch (m_node->arrayMode().type()) {
1543             case Array::Int32:
1544             case Array::Contiguous: {
1545                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
1546
1547                 if (m_node->arrayMode().type() == Array::Int32)
1548                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
1549
1550                 TypedPointer elementPointer = m_out.baseIndex(
1551                     m_node->arrayMode().type() == Array::Int32 ?
1552                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
1553                     storage, m_out.zeroExt(index, m_out.intPtr),
1554                     m_state.forNode(child2).m_value);
1555
1556                 if (m_node->op() == PutByValAlias) {
1557                     m_out.store64(value, elementPointer);
1558                     break;
1559                 }
1560
1561                 contiguousPutByValOutOfBounds(
1562                     codeBlock()->isStrictMode()
1563                     ? operationPutByValBeyondArrayBoundsStrict
1564                     : operationPutByValBeyondArrayBoundsNonStrict,
1565                     base, storage, index, value, continuation);
1566
1567                 m_out.store64(value, elementPointer);
1568                 break;
1569             }
1570
1571             case Array::Double: {
1572                 LValue value = lowDouble(child3);
1573
1574                 FTL_TYPE_CHECK(
1575                     doubleValue(value), child3, SpecFullRealNumber,
1576                     m_out.doubleNotEqualOrUnordered(value, value));
1577
1578                 TypedPointer elementPointer = m_out.baseIndex(
1579                     m_heaps.indexedDoubleProperties,
1580                     storage, m_out.zeroExt(index, m_out.intPtr),
1581                     m_state.forNode(child2).m_value);
1582
1583                 if (m_node->op() == PutByValAlias) {
1584                     m_out.storeDouble(value, elementPointer);
1585                     break;
1586                 }
1587
1588                 contiguousPutByValOutOfBounds(
1589                     codeBlock()->isStrictMode()
1590                     ? operationPutDoubleByValBeyondArrayBoundsStrict
1591                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
1592                     base, storage, index, value, continuation);
1593
1594                 m_out.storeDouble(value, elementPointer);
1595                 break;
1596             }
1597
1598             default:
1599                 RELEASE_ASSERT_NOT_REACHED();
1600             }
1601
1602             m_out.jump(continuation);
1603             m_out.appendTo(continuation, outerLastNext);
1604             return;
1605         }
1606
1607         default:
1608             TypedArrayType type = m_node->arrayMode().typedArrayType();
1609
1610             if (isTypedView(type)) {
1611                 if (m_node->op() == PutByVal) {
1612                     speculate(
1613                         OutOfBounds, noValue(), 0,
1614                         m_out.aboveOrEqual(
1615                             index, m_out.load32(base, m_heaps.JSArrayBufferView_length)));
1616                 }
1617
1618                 TypedPointer pointer = TypedPointer(
1619                     m_heaps.typedArrayProperties,
1620                     m_out.add(
1621                         storage,
1622                         m_out.shl(
1623                             m_out.zeroExt(index, m_out.intPtr),
1624                             m_out.constIntPtr(logElementSize(type)))));
1625
1626                 if (isInt(type)) {
1627                     LValue intValue;
1628                     switch (child3.useKind()) {
1629                     case Int32Use: {
1630                         intValue = lowInt32(child3);
1631                         if (isClamped(type)) {
1632                             ASSERT(elementSize(type) == 1);
1633
1634                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
1635                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
1636
1637                             Vector<ValueFromBlock, 2> intValues;
1638                             intValues.append(m_out.anchor(m_out.int32Zero));
1639                             m_out.branch(
1640                                 m_out.lessThan(intValue, m_out.int32Zero),
1641                                 continuation, atLeastZero);
1642
1643                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
1644
1645                             intValues.append(m_out.anchor(m_out.select(
1646                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
1647                                 m_out.constInt32(255),
1648                                 intValue)));
1649                             m_out.jump(continuation);
1650
1651                             m_out.appendTo(continuation, lastNext);
1652                             intValue = m_out.phi(m_out.int32, intValues);
1653                         }
1654                         break;
1655                     }
1656
1657                     case NumberUse: {
1658                         LValue doubleValue = lowDouble(child3);
1659
1660                         if (isClamped(type)) {
1661                             ASSERT(elementSize(type) == 1);
1662
1663                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
1664                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
1665                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
1666
1667                             Vector<ValueFromBlock, 3> intValues;
1668                             intValues.append(m_out.anchor(m_out.int32Zero));
1669                             m_out.branch(
1670                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
1671                                 continuation, atLeastZero);
1672
1673                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
1674                             intValues.append(m_out.anchor(m_out.constInt32(255)));
1675                             m_out.branch(
1676                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
1677                                 continuation, withinRange);
1678
1679                             m_out.appendTo(withinRange, continuation);
1680                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
1681                             m_out.jump(continuation);
1682
1683                             m_out.appendTo(continuation, lastNext);
1684                             intValue = m_out.phi(m_out.int32, intValues);
1685                         } else if (isSigned(type))
1686                             intValue = doubleToInt32(doubleValue);
1687                         else
1688                             intValue = doubleToUInt32(doubleValue);
1689                         break;
1690                     }
1691
1692                     default:
1693                         RELEASE_ASSERT_NOT_REACHED();
1694                     }
1695
1696                     switch (elementSize(type)) {
1697                     case 1:
1698                         m_out.store8(m_out.intCast(intValue, m_out.int8), pointer);
1699                         break;
1700                     case 2:
1701                         m_out.store16(m_out.intCast(intValue, m_out.int16), pointer);
1702                         break;
1703                     case 4:
1704                         m_out.store32(intValue, pointer);
1705                         break;
1706                     default:
1707                         RELEASE_ASSERT_NOT_REACHED();
1708                     }
1709
1710                     return;
1711                 }
1712
1713                 ASSERT(isFloat(type));
1714
1715                 LValue value = lowDouble(child3);
1716                 switch (type) {
1717                 case TypeFloat32:
1718                     m_out.storeFloat(m_out.fpCast(value, m_out.floatType), pointer);
1719                     break;
1720                 case TypeFloat64:
1721                     m_out.storeDouble(value, pointer);
1722                     break;
1723                 default:
1724                     RELEASE_ASSERT_NOT_REACHED();
1725                 }
1726                 return;
1727             }
1728
1729             RELEASE_ASSERT_NOT_REACHED();
1730             break;
1731         }
1732     }
1733
1734     void compileGetByOffset()
1735     {
1736         StorageAccessData& data =
1737             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1738
1739         setJSValue(
1740             m_out.load64(
1741                 m_out.address(
1742                     m_heaps.properties[data.identifierNumber],
1743                     lowStorage(m_node->child1()),
1744                     offsetRelativeToBase(data.offset))));
1745     }
1746
1747     void compilePutByOffset()
1748     {
1749         StorageAccessData& data =
1750             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
1751
1752         m_out.store64(
1753             lowJSValue(m_node->child3()),
1754             m_out.address(
1755                 m_heaps.properties[data.identifierNumber],
1756                 lowStorage(m_node->child1()),
1757                 offsetRelativeToBase(data.offset)));
1758     }
1759
1760     void compileGetGlobalVar()
1761     {
1762         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
1763     }
1764
1765     void compilePutGlobalVar()
1766     {
1767         m_out.store64(
1768             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
1769     }
1770
1771     void compileGlobalVarWatchpoint()
1772     {
1773         // FIXME: Implement watchpoints.
1774         // https://bugs.webkit.org/show_bug.cgi?id=113647
1775     }
1776
1777     void compileGetMyScope()
1778     {
1779         setJSValue(m_out.loadPtr(addressFor(
1780             m_node->codeOrigin.stackOffset() + JSStack::ScopeChain)));
1781     }
1782
1783     void compileSkipScope()
1784     {
1785         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
1786     }
1787
1788     void compileGetClosureRegisters()
1789     {
1790         setStorage(m_out.loadPtr(
1791             lowCell(m_node->child1()), m_heaps.JSVariableObject_registers));
1792     }
1793
1794     void compileGetClosureVar()
1795     {
1796         setJSValue(m_out.load64(
1797             addressFor(lowStorage(m_node->child1()), m_node->varNumber())));
1798     }
1799
1800     void compilePutClosureVar()
1801     {
1802         m_out.store64(
1803             lowJSValue(m_node->child3()),
1804             addressFor(lowStorage(m_node->child2()), m_node->varNumber()));
1805     }
1806
1807     void compileCompareEq()
1808     {
1809         if (m_node->isBinaryUseKind(Int32Use)
1810             || m_node->isBinaryUseKind(MachineIntUse)
1811             || m_node->isBinaryUseKind(NumberUse)
1812             || m_node->isBinaryUseKind(ObjectUse)) {
1813             compileCompareStrictEq();
1814             return;
1815         }
1816
1817         RELEASE_ASSERT_NOT_REACHED();
1818     }
1819
1820     void compileCompareEqConstant()
1821     {
1822         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
1823         masqueradesAsUndefinedWatchpointIfIsStillValid();
1824         setBoolean(
1825             equalNullOrUndefined(
1826                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
1827     }
1828
1829     void compileCompareStrictEq()
1830     {
1831         if (m_node->isBinaryUseKind(Int32Use)) {
1832             setBoolean(
1833                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1834             return;
1835         }
1836
1837         if (m_node->isBinaryUseKind(MachineIntUse)) {
1838             Int52Kind kind;
1839             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1840             LValue right = lowInt52(m_node->child2(), kind);
1841             setBoolean(m_out.equal(left, right));
1842             return;
1843         }
1844
1845         if (m_node->isBinaryUseKind(NumberUse)) {
1846             setBoolean(
1847                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1848             return;
1849         }
1850
1851         if (m_node->isBinaryUseKind(ObjectUse)) {
1852             masqueradesAsUndefinedWatchpointIfIsStillValid();
1853             setBoolean(
1854                 m_out.equal(
1855                     lowNonNullObject(m_node->child1()),
1856                     lowNonNullObject(m_node->child2())));
1857             return;
1858         }
1859
1860         RELEASE_ASSERT_NOT_REACHED();
1861     }
1862
1863     void compileCompareStrictEqConstant()
1864     {
1865         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
1866
1867         if (constant.isUndefinedOrNull()
1868             && !masqueradesAsUndefinedWatchpointIfIsStillValid()) {
1869             if (constant.isNull()) {
1870                 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull));
1871                 return;
1872             }
1873
1874             ASSERT(constant.isUndefined());
1875             setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
1876             return;
1877         }
1878
1879         setBoolean(
1880             m_out.equal(
1881                 lowJSValue(m_node->child1()),
1882                 m_out.constInt64(JSValue::encode(constant))));
1883     }
1884
1885     void compileCompareLess()
1886     {
1887         if (m_node->isBinaryUseKind(Int32Use)) {
1888             setBoolean(
1889                 m_out.lessThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1890             return;
1891         }
1892
1893         if (m_node->isBinaryUseKind(MachineIntUse)) {
1894             Int52Kind kind;
1895             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1896             LValue right = lowInt52(m_node->child2(), kind);
1897             setBoolean(m_out.lessThan(left, right));
1898             return;
1899         }
1900
1901         if (m_node->isBinaryUseKind(NumberUse)) {
1902             setBoolean(
1903                 m_out.doubleLessThan(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1904             return;
1905         }
1906
1907         RELEASE_ASSERT_NOT_REACHED();
1908     }
1909
1910     void compileCompareLessEq()
1911     {
1912         if (m_node->isBinaryUseKind(Int32Use)) {
1913             setBoolean(
1914                 m_out.lessThanOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1915             return;
1916         }
1917
1918         if (m_node->isBinaryUseKind(MachineIntUse)) {
1919             Int52Kind kind;
1920             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1921             LValue right = lowInt52(m_node->child2(), kind);
1922             setBoolean(m_out.lessThanOrEqual(left, right));
1923             return;
1924         }
1925
1926         if (m_node->isBinaryUseKind(NumberUse)) {
1927             setBoolean(
1928                 m_out.doubleLessThanOrEqual(
1929                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1930             return;
1931         }
1932
1933         RELEASE_ASSERT_NOT_REACHED();
1934     }
1935
1936     void compileCompareGreater()
1937     {
1938         if (m_node->isBinaryUseKind(Int32Use)) {
1939             setBoolean(
1940                 m_out.greaterThan(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1941             return;
1942         }
1943
1944         if (m_node->isBinaryUseKind(MachineIntUse)) {
1945             Int52Kind kind;
1946             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1947             LValue right = lowInt52(m_node->child2(), kind);
1948             setBoolean(m_out.greaterThan(left, right));
1949             return;
1950         }
1951
1952         if (m_node->isBinaryUseKind(NumberUse)) {
1953             setBoolean(
1954                 m_out.doubleGreaterThan(
1955                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1956             return;
1957         }
1958
1959         RELEASE_ASSERT_NOT_REACHED();
1960     }
1961
1962     void compileCompareGreaterEq()
1963     {
1964         if (m_node->isBinaryUseKind(Int32Use)) {
1965             setBoolean(
1966                 m_out.greaterThanOrEqual(
1967                     lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1968             return;
1969         }
1970
1971         if (m_node->isBinaryUseKind(MachineIntUse)) {
1972             Int52Kind kind;
1973             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1974             LValue right = lowInt52(m_node->child2(), kind);
1975             setBoolean(m_out.greaterThanOrEqual(left, right));
1976             return;
1977         }
1978
1979         if (m_node->isBinaryUseKind(NumberUse)) {
1980             setBoolean(
1981                 m_out.doubleGreaterThanOrEqual(
1982                     lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1983             return;
1984         }
1985
1986         RELEASE_ASSERT_NOT_REACHED();
1987     }
1988
1989     void compileLogicalNot()
1990     {
1991         setBoolean(m_out.bitNot(boolify(m_node->child1())));
1992     }
1993
1994     void compileCallOrConstruct()
1995     {
1996         // FIXME: This is unacceptably slow.
1997         // https://bugs.webkit.org/show_bug.cgi?id=113621
1998
1999         J_DFGOperation_E function =
2000             m_node->op() == Call ? operationFTLCall : operationFTLConstruct;
2001
2002         int dummyThisArgument = m_node->op() == Call ? 0 : 1;
2003
2004         int numPassedArgs = m_node->numChildren() - 1;
2005
2006         LValue calleeFrame = m_out.add(
2007             m_callFrame,
2008             m_out.constIntPtr(sizeof(Register) * virtualRegisterForLocal(codeBlock()->m_numCalleeRegisters).offset()));
2009
2010         m_out.store32(
2011             m_out.constInt32(numPassedArgs + dummyThisArgument),
2012             payloadFor(calleeFrame, JSStack::ArgumentCount));
2013         m_out.store64(m_callFrame, addressFor(calleeFrame, JSStack::CallerFrame));
2014         m_out.store64(
2015             lowJSValue(m_graph.varArgChild(m_node, 0)),
2016             addressFor(calleeFrame, JSStack::Callee));
2017
2018         for (int i = 0; i < numPassedArgs; ++i) {
2019             m_out.store64(
2020                 lowJSValue(m_graph.varArgChild(m_node, 1 + i)),
2021                 addressFor(calleeFrame, virtualRegisterForArgument(i + dummyThisArgument).offset()));
2022         }
2023
2024         setJSValue(vmCall(m_out.operation(function), calleeFrame));
2025     }
2026
2027     void compileJump()
2028     {
2029         m_out.jump(lowBlock(m_node->takenBlock()));
2030     }
2031
2032     void compileBranch()
2033     {
2034         m_out.branch(
2035             boolify(m_node->child1()),
2036             lowBlock(m_node->takenBlock()),
2037             lowBlock(m_node->notTakenBlock()));
2038     }
2039
2040     void compileSwitch()
2041     {
2042         SwitchData* data = m_node->switchData();
2043         switch (data->kind) {
2044         case SwitchImm: {
2045             Vector<ValueFromBlock, 2> intValues;
2046             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
2047
2048             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
2049
2050             switch (m_node->child1().useKind()) {
2051             case Int32Use: {
2052                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
2053                 m_out.jump(switchOnInts);
2054                 break;
2055             }
2056
2057             case UntypedUse: {
2058                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
2059                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
2060                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
2061
2062                 LValue boxedValue = lowJSValue(m_node->child1());
2063                 m_out.branch(isNotInt32(boxedValue), isNotInt, isInt);
2064
2065                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
2066
2067                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
2068                 m_out.jump(switchOnInts);
2069
2070                 m_out.appendTo(isNotInt, isDouble);
2071                 m_out.branch(
2072                     isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble);
2073
2074                 m_out.appendTo(isDouble, innerLastNext);
2075                 LValue doubleValue = unboxDouble(boxedValue);
2076                 LValue intInDouble = m_out.fpToInt32(doubleValue);
2077                 intValues.append(m_out.anchor(intInDouble));
2078                 m_out.branch(
2079                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
2080                     switchOnInts, lowBlock(data->fallThrough));
2081                 break;
2082             }
2083
2084             default:
2085                 RELEASE_ASSERT_NOT_REACHED();
2086                 break;
2087             }
2088
2089             m_out.appendTo(switchOnInts, lastNext);
2090             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
2091             return;
2092         }
2093
2094         case SwitchChar: {
2095             LValue stringValue;
2096
2097             switch (m_node->child1().useKind()) {
2098             case StringUse: {
2099                 stringValue = lowString(m_node->child1());
2100                 break;
2101             }
2102
2103             case UntypedUse: {
2104                 LValue unboxedValue = lowJSValue(m_node->child1());
2105
2106                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
2107                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
2108
2109                 m_out.branch(
2110                     isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase);
2111
2112                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
2113                 LValue cellValue = unboxedValue;
2114                 m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase);
2115
2116                 m_out.appendTo(isStringCase, lastNext);
2117                 stringValue = cellValue;
2118                 break;
2119             }
2120
2121             default:
2122                 RELEASE_ASSERT_NOT_REACHED();
2123                 break;
2124             }
2125
2126             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
2127             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
2128             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
2129             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
2130             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
2131             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
2132
2133             m_out.branch(
2134                 m_out.notEqual(
2135                     m_out.load32(stringValue, m_heaps.JSString_length),
2136                     m_out.int32One),
2137                 lowBlock(data->fallThrough), lengthIs1);
2138
2139             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
2140             Vector<ValueFromBlock, 2> values;
2141             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
2142             values.append(m_out.anchor(fastValue));
2143             m_out.branch(m_out.isNull(fastValue), needResolution, resolved);
2144
2145             m_out.appendTo(needResolution, resolved);
2146             values.append(m_out.anchor(
2147                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
2148             m_out.jump(resolved);
2149
2150             m_out.appendTo(resolved, is8Bit);
2151             LValue value = m_out.phi(m_out.intPtr, values);
2152             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
2153             m_out.branch(
2154                 m_out.testNonZero32(
2155                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
2156                     m_out.constInt32(StringImpl::flagIs8Bit())),
2157                 is8Bit, is16Bit);
2158
2159             Vector<ValueFromBlock, 2> characters;
2160             m_out.appendTo(is8Bit, is16Bit);
2161             characters.append(m_out.anchor(
2162                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
2163             m_out.jump(continuation);
2164
2165             m_out.appendTo(is16Bit, continuation);
2166             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
2167             m_out.jump(continuation);
2168
2169             m_out.appendTo(continuation, lastNext);
2170             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
2171             return;
2172         }
2173
2174         case SwitchString:
2175             RELEASE_ASSERT_NOT_REACHED();
2176             break;
2177         }
2178
2179         RELEASE_ASSERT_NOT_REACHED();
2180     }
2181
2182     void compileReturn()
2183     {
2184         // FIXME: have a real epilogue when we switch to using our calling convention.
2185         // https://bugs.webkit.org/show_bug.cgi?id=113621
2186         m_out.ret(lowJSValue(m_node->child1()));
2187     }
2188
2189     void compileForceOSRExit()
2190     {
2191         terminate(InadequateCoverage);
2192     }
2193
2194     LValue boolify(Edge edge)
2195     {
2196         switch (edge.useKind()) {
2197         case BooleanUse:
2198             return lowBoolean(m_node->child1());
2199         case Int32Use:
2200             return m_out.notZero32(lowInt32(m_node->child1()));
2201         case NumberUse:
2202             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
2203         case ObjectOrOtherUse:
2204             return m_out.bitNot(
2205                 equalNullOrUndefined(
2206                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
2207                     ManualOperandSpeculation));
2208         default:
2209             RELEASE_ASSERT_NOT_REACHED();
2210             return 0;
2211         }
2212     }
2213
2214     enum StringOrObjectMode {
2215         AllCellsAreFalse,
2216         CellCaseSpeculatesObject
2217     };
2218     enum EqualNullOrUndefinedMode {
2219         EqualNull,
2220         EqualUndefined,
2221         EqualNullOrUndefined,
2222         SpeculateNullOrUndefined
2223     };
2224     LValue equalNullOrUndefined(
2225         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
2226         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
2227     {
2228         bool validWatchpoint = masqueradesAsUndefinedWatchpointIfIsStillValid();
2229
2230         LValue value = lowJSValue(edge, operandMode);
2231
2232         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
2233         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
2234         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
2235
2236         m_out.branch(isNotCell(value), primitiveCase, cellCase);
2237
2238         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
2239
2240         Vector<ValueFromBlock, 3> results;
2241
2242         switch (cellMode) {
2243         case AllCellsAreFalse:
2244             break;
2245         case CellCaseSpeculatesObject:
2246             FTL_TYPE_CHECK(
2247                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
2248                 m_out.equal(
2249                     m_out.loadPtr(value, m_heaps.JSCell_structure),
2250                     m_out.constIntPtr(vm().stringStructure.get())));
2251             break;
2252         }
2253
2254         if (validWatchpoint) {
2255             results.append(m_out.anchor(m_out.booleanFalse));
2256             m_out.jump(continuation);
2257         } else {
2258             LBasicBlock masqueradesCase =
2259                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
2260
2261             LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure);
2262
2263             results.append(m_out.anchor(m_out.booleanFalse));
2264
2265             m_out.branch(
2266                 m_out.testNonZero8(
2267                     m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
2268                     m_out.constInt8(MasqueradesAsUndefined)),
2269                 masqueradesCase, continuation);
2270
2271             m_out.appendTo(masqueradesCase, primitiveCase);
2272
2273             results.append(m_out.anchor(
2274                 m_out.equal(
2275                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)),
2276                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
2277             m_out.jump(continuation);
2278         }
2279
2280         m_out.appendTo(primitiveCase, continuation);
2281
2282         LValue primitiveResult;
2283         switch (primitiveMode) {
2284         case EqualNull:
2285             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
2286             break;
2287         case EqualUndefined:
2288             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
2289             break;
2290         case EqualNullOrUndefined:
2291             primitiveResult = m_out.equal(
2292                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
2293                 m_out.constInt64(ValueNull));
2294             break;
2295         case SpeculateNullOrUndefined:
2296             FTL_TYPE_CHECK(
2297                 jsValueValue(value), edge, SpecCell | SpecOther,
2298                 m_out.notEqual(
2299                     m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
2300                     m_out.constInt64(ValueNull)));
2301             primitiveResult = m_out.booleanTrue;
2302             break;
2303         }
2304         results.append(m_out.anchor(primitiveResult));
2305         m_out.jump(continuation);
2306
2307         m_out.appendTo(continuation, lastNext);
2308
2309         return m_out.phi(m_out.boolean, results);
2310     }
2311
2312     template<typename FunctionType>
2313     void contiguousPutByValOutOfBounds(
2314         FunctionType slowPathFunction,
2315         LValue base, LValue storage, LValue index, LValue value,
2316         LBasicBlock continuation)
2317     {
2318         LValue isNotInBounds = m_out.aboveOrEqual(
2319             index, m_out.load32(storage, m_heaps.Butterfly_publicLength));
2320         if (m_node->arrayMode().isInBounds())
2321             speculate(StoreToHoleOrOutOfBounds, noValue(), 0, isNotInBounds);
2322         else {
2323             LBasicBlock notInBoundsCase =
2324                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
2325             LBasicBlock performStore =
2326                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
2327
2328             m_out.branch(isNotInBounds, notInBoundsCase, performStore);
2329
2330             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
2331
2332             LValue isOutOfBounds = m_out.aboveOrEqual(
2333                 index, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
2334
2335             if (!m_node->arrayMode().isOutOfBounds())
2336                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
2337             else {
2338                 LBasicBlock outOfBoundsCase =
2339                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
2340                 LBasicBlock holeCase =
2341                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
2342
2343                 m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase);
2344
2345                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
2346
2347                 vmCall(
2348                     m_out.operation(slowPathFunction),
2349                     m_callFrame, base, index, value);
2350
2351                 m_out.jump(continuation);
2352
2353                 m_out.appendTo(holeCase, innerLastNext);
2354             }
2355
2356             m_out.store32(
2357                 m_out.add(index, m_out.int32One),
2358                 storage, m_heaps.Butterfly_publicLength);
2359
2360             m_out.jump(performStore);
2361             m_out.appendTo(performStore, lastNext);
2362         }
2363     }
2364
2365     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
2366     {
2367         Vector<SwitchCase> cases;
2368         for (unsigned i = 0; i < data->cases.size(); ++i) {
2369             cases.append(SwitchCase(
2370                 constInt(type, data->cases[i].value.switchLookupValue()),
2371                 lowBlock(data->cases[i].target)));
2372         }
2373
2374         m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough));
2375     }
2376
2377     LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
2378     {
2379         // FIXME: Optimize double-to-int conversions.
2380         // <rdar://problem/14938465>
2381
2382         LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
2383         LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
2384         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
2385         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
2386
2387         Vector<ValueFromBlock, 2> results;
2388
2389         m_out.branch(
2390             m_out.greaterThanOrEqual(doubleValue, m_out.constDouble(low)),
2391             greatEnough, slowPath);
2392
2393         LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
2394         m_out.branch(
2395             m_out.lessThanOrEqual(doubleValue, m_out.constDouble(high)),
2396             withinRange, slowPath);
2397
2398         m_out.appendTo(withinRange, slowPath);
2399         LValue fastResult;
2400         if (isSigned)
2401             fastResult = m_out.fpToInt32(doubleValue);
2402         else
2403             fastResult = m_out.fpToUInt32(doubleValue);
2404         results.append(m_out.anchor(fastResult));
2405         m_out.jump(continuation);
2406
2407         m_out.appendTo(slowPath, continuation);
2408         results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
2409         m_out.jump(continuation);
2410
2411         m_out.appendTo(continuation, lastNext);
2412         return m_out.phi(m_out.int32, results);
2413     }
2414
2415     LValue doubleToInt32(LValue doubleValue)
2416     {
2417         double limit = pow(2, 31) - 1;
2418         return doubleToInt32(doubleValue, -limit, limit);
2419     }
2420
2421     LValue doubleToUInt32(LValue doubleValue)
2422     {
2423         return doubleToInt32(doubleValue, 0, pow(2, 32) - 1, false);
2424     }
2425
2426     void speculateBackward(
2427         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
2428     {
2429         appendOSRExit(
2430             kind, lowValue, highValue, failCondition, BackwardSpeculation, FormattedValue());
2431     }
2432
2433     void speculateForward(
2434         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
2435         const FormattedValue& recovery)
2436     {
2437         appendOSRExit(
2438             kind, lowValue, highValue, failCondition, ForwardSpeculation, recovery);
2439     }
2440
2441     void speculate(
2442         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
2443     {
2444         appendOSRExit(
2445             kind, lowValue, highValue, failCondition, m_direction, FormattedValue());
2446     }
2447
2448     void terminate(ExitKind kind)
2449     {
2450         speculate(kind, noValue(), 0, m_out.booleanTrue);
2451     }
2452
2453     void backwardTypeCheck(
2454         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2455         LValue failCondition)
2456     {
2457         appendTypeCheck(
2458             lowValue, highValue, typesPassedThrough, failCondition, BackwardSpeculation,
2459             FormattedValue());
2460     }
2461
2462     void forwardTypeCheck(
2463         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2464         LValue failCondition)
2465     {
2466         appendTypeCheck(
2467             lowValue, highValue, typesPassedThrough, failCondition, ForwardSpeculation,
2468             FormattedValue());
2469     }
2470
2471     void typeCheck(
2472         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2473         LValue failCondition)
2474     {
2475         appendTypeCheck(
2476             lowValue, highValue, typesPassedThrough, failCondition, m_direction,
2477             FormattedValue());
2478     }
2479
2480     void appendTypeCheck(
2481         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
2482         LValue failCondition, SpeculationDirection direction, FormattedValue recovery)
2483     {
2484         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
2485             return;
2486         ASSERT(mayHaveTypeCheck(highValue.useKind()));
2487         appendOSRExit(BadType, lowValue, highValue.node(), failCondition, direction, recovery);
2488         m_interpreter.filter(highValue, typesPassedThrough);
2489     }
2490
2491     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2492     {
2493         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
2494
2495         if (edge->hasConstant()) {
2496             JSValue value = m_graph.valueOfJSConstant(edge.node());
2497             if (!value.isInt32()) {
2498                 terminate(Uncountable);
2499                 return m_out.int32Zero;
2500             }
2501             return m_out.constInt32(value.asInt32());
2502         }
2503
2504         LoweredNodeValue value = m_int32Values.get(edge.node());
2505         if (isValid(value))
2506             return value.value();
2507
2508         value = m_strictInt52Values.get(edge.node());
2509         if (isValid(value))
2510             return strictInt52ToInt32(edge, value.value());
2511
2512         value = m_int52Values.get(edge.node());
2513         if (isValid(value))
2514             return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
2515
2516         value = m_jsValueValues.get(edge.node());
2517         if (isValid(value)) {
2518             LValue boxedResult = value.value();
2519             FTL_TYPE_CHECK(
2520                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
2521             LValue result = unboxInt32(boxedResult);
2522             setInt32(edge.node(), result);
2523             return result;
2524         }
2525
2526         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
2527         terminate(Uncountable);
2528         return m_out.int32Zero;
2529     }
2530
2531     enum Int52Kind { StrictInt52, Int52 };
2532     LValue lowInt52(Edge edge, Int52Kind kind, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2533     {
2534         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == MachineIntUse);
2535
2536         if (edge->hasConstant()) {
2537             JSValue value = m_graph.valueOfJSConstant(edge.node());
2538             if (!value.isMachineInt()) {
2539                 terminate(Uncountable);
2540                 return m_out.int64Zero;
2541             }
2542             int64_t result = value.asMachineInt();
2543             if (kind == Int52)
2544                 result <<= JSValue::int52ShiftAmount;
2545             return m_out.constInt64(result);
2546         }
2547
2548         LoweredNodeValue value;
2549
2550         switch (kind) {
2551         case Int52:
2552             value = m_int52Values.get(edge.node());
2553             if (isValid(value))
2554                 return value.value();
2555
2556             value = m_strictInt52Values.get(edge.node());
2557             if (isValid(value))
2558                 return strictInt52ToInt52(value.value());
2559             break;
2560
2561         case StrictInt52:
2562             value = m_strictInt52Values.get(edge.node());
2563             if (isValid(value))
2564                 return value.value();
2565
2566             value = m_int52Values.get(edge.node());
2567             if (isValid(value))
2568                 return int52ToStrictInt52(value.value());
2569             break;
2570         }
2571
2572         value = m_int32Values.get(edge.node());
2573         if (isValid(value)) {
2574             return setInt52WithStrictValue(
2575                 edge.node(), m_out.signExt(value.value(), m_out.int64), kind);
2576         }
2577
2578         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt52));
2579
2580         value = m_jsValueValues.get(edge.node());
2581         if (isValid(value)) {
2582             LValue boxedResult = value.value();
2583             FTL_TYPE_CHECK(
2584                 jsValueValue(boxedResult), edge, SpecMachineInt, isNotInt32(boxedResult));
2585             return setInt52WithStrictValue(
2586                 edge.node(), m_out.signExt(unboxInt32(boxedResult), m_out.int64), kind);
2587         }
2588
2589         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecMachineInt));
2590         terminate(Uncountable);
2591         return m_out.int64Zero;
2592     }
2593
2594     LValue lowInt52(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2595     {
2596         return lowInt52(edge, Int52, mode);
2597     }
2598
2599     LValue lowStrictInt52(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2600     {
2601         return lowInt52(edge, StrictInt52, mode);
2602     }
2603
2604     bool betterUseStrictInt52(Node* node)
2605     {
2606         return !isValid(m_int52Values.get(node));
2607     }
2608     bool betterUseStrictInt52(Edge edge)
2609     {
2610         return betterUseStrictInt52(edge.node());
2611     }
2612     template<typename T>
2613     Int52Kind bestInt52Kind(T node)
2614     {
2615         return betterUseStrictInt52(node) ? StrictInt52 : Int52;
2616     }
2617     Int52Kind opposite(Int52Kind kind)
2618     {
2619         switch (kind) {
2620         case Int52:
2621             return StrictInt52;
2622         case StrictInt52:
2623             return Int52;
2624         }
2625         RELEASE_ASSERT_NOT_REACHED();
2626     }
2627
2628     LValue lowWhicheverInt52(Edge edge, Int52Kind& kind, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2629     {
2630         kind = bestInt52Kind(edge);
2631         return lowInt52(edge, kind, mode);
2632     }
2633
2634     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2635     {
2636         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
2637
2638         if (edge->op() == JSConstant) {
2639             JSValue value = m_graph.valueOfJSConstant(edge.node());
2640             if (!value.isCell()) {
2641                 terminate(Uncountable);
2642                 return m_out.intPtrZero;
2643             }
2644             return m_out.constIntPtr(value.asCell());
2645         }
2646
2647         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2648         if (isValid(value)) {
2649             LValue uncheckedValue = value.value();
2650             FTL_TYPE_CHECK(
2651                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
2652             return uncheckedValue;
2653         }
2654
2655         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
2656         terminate(Uncountable);
2657         return m_out.intPtrZero;
2658     }
2659
2660     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2661     {
2662         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2663
2664         LValue result = lowCell(edge, mode);
2665         speculateObject(edge, result);
2666         return result;
2667     }
2668
2669     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2670     {
2671         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse);
2672
2673         LValue result = lowCell(edge, mode);
2674         speculateString(edge, result);
2675         return result;
2676     }
2677
2678     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2679     {
2680         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
2681
2682         LValue result = lowCell(edge, mode);
2683         speculateNonNullObject(edge, result);
2684         return result;
2685     }
2686
2687     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2688     {
2689         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
2690
2691         if (edge->hasConstant()) {
2692             JSValue value = m_graph.valueOfJSConstant(edge.node());
2693             if (!value.isBoolean()) {
2694                 terminate(Uncountable);
2695                 return m_out.booleanFalse;
2696             }
2697             return m_out.constBool(value.asBoolean());
2698         }
2699
2700         LoweredNodeValue value = m_booleanValues.get(edge.node());
2701         if (isValid(value))
2702             return value.value();
2703
2704         value = m_jsValueValues.get(edge.node());
2705         if (isValid(value)) {
2706             LValue unboxedResult = value.value();
2707             FTL_TYPE_CHECK(
2708                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
2709             LValue result = unboxBoolean(unboxedResult);
2710             setBoolean(edge.node(), result);
2711             return result;
2712         }
2713
2714         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
2715         terminate(Uncountable);
2716         return m_out.booleanFalse;
2717     }
2718
2719     LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2720     {
2721         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind()));
2722
2723         if (edge->hasConstant()) {
2724             JSValue value = m_graph.valueOfJSConstant(edge.node());
2725             if (!value.isNumber()) {
2726                 terminate(Uncountable);
2727                 return m_out.doubleZero;
2728             }
2729             return m_out.constDouble(value.asNumber());
2730         }
2731
2732         LoweredNodeValue value = m_doubleValues.get(edge.node());
2733         if (isValid(value))
2734             return value.value();
2735
2736         value = m_int32Values.get(edge.node());
2737         if (isValid(value)) {
2738             LValue result = m_out.intToDouble(value.value());
2739             setDouble(edge.node(), result);
2740             return result;
2741         }
2742
2743         value = m_strictInt52Values.get(edge.node());
2744         if (isValid(value))
2745             return strictInt52ToDouble(edge, value.value());
2746
2747         value = m_int52Values.get(edge.node());
2748         if (isValid(value))
2749             return strictInt52ToDouble(edge, int52ToStrictInt52(value.value()));
2750
2751         value = m_jsValueValues.get(edge.node());
2752         if (isValid(value)) {
2753             LValue boxedResult = value.value();
2754
2755             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case"));
2756             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case"));
2757             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation"));
2758
2759             m_out.branch(isNotInt32(boxedResult), doubleCase, intCase);
2760
2761             LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
2762
2763             ValueFromBlock intToDouble = m_out.anchor(
2764                 m_out.intToDouble(unboxInt32(boxedResult)));
2765             m_out.jump(continuation);
2766
2767             m_out.appendTo(doubleCase, continuation);
2768
2769             FTL_TYPE_CHECK(
2770                 jsValueValue(boxedResult), edge, SpecFullNumber, isCellOrMisc(boxedResult));
2771
2772             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult));
2773             m_out.jump(continuation);
2774
2775             m_out.appendTo(continuation, lastNext);
2776
2777             LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
2778
2779             setDouble(edge.node(), result);
2780             return result;
2781         }
2782
2783         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecFullNumber));
2784         terminate(Uncountable);
2785         return m_out.doubleZero;
2786     }
2787
2788     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2789     {
2790         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2791
2792         if (edge->hasConstant())
2793             return m_out.constInt64(JSValue::encode(m_graph.valueOfJSConstant(edge.node())));
2794
2795         LoweredNodeValue value = m_jsValueValues.get(edge.node());
2796         if (isValid(value))
2797             return value.value();
2798
2799         value = m_int32Values.get(edge.node());
2800         if (isValid(value)) {
2801             LValue result = boxInt32(value.value());
2802             setJSValue(edge.node(), result);
2803             return result;
2804         }
2805
2806         value = m_strictInt52Values.get(edge.node());
2807         if (isValid(value))
2808             return strictInt52ToJSValue(value.value());
2809
2810         value = m_int52Values.get(edge.node());
2811         if (isValid(value))
2812             return strictInt52ToJSValue(int52ToStrictInt52(value.value()));
2813
2814         value = m_booleanValues.get(edge.node());
2815         if (isValid(value)) {
2816             LValue result = boxBoolean(value.value());
2817             setJSValue(edge.node(), result);
2818             return result;
2819         }
2820
2821         value = m_doubleValues.get(edge.node());
2822         if (isValid(value)) {
2823             LValue result = boxDouble(value.value());
2824             setJSValue(edge.node(), result);
2825             return result;
2826         }
2827
2828         RELEASE_ASSERT_NOT_REACHED();
2829         return 0;
2830     }
2831
2832     LValue lowStorage(Edge edge)
2833     {
2834         LoweredNodeValue value = m_storageValues.get(edge.node());
2835         if (isValid(value))
2836             return value.value();
2837
2838         LValue result = lowCell(edge);
2839         setStorage(edge.node(), result);
2840         return result;
2841     }
2842
2843     LValue strictInt52ToInt32(Edge edge, LValue value)
2844     {
2845         LValue result = m_out.castToInt32(value);
2846         FTL_TYPE_CHECK(
2847             noValue(), edge, SpecInt32,
2848             m_out.notEqual(m_out.signExt(result, m_out.int64), value));
2849         setInt32(edge.node(), result);
2850         return result;
2851     }
2852
2853     LValue strictInt52ToDouble(Edge edge, LValue value)
2854     {
2855         LValue result = m_out.intToDouble(value);
2856         setDouble(edge.node(), result);
2857         return result;
2858     }
2859
2860     LValue strictInt52ToJSValue(LValue value)
2861     {
2862         LBasicBlock isInt32 = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isInt32 case"));
2863         LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isDouble case"));
2864         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue continuation"));
2865
2866         Vector<ValueFromBlock, 2> results;
2867
2868         LValue int32Value = m_out.castToInt32(value);
2869         m_out.branch(
2870             m_out.equal(m_out.signExt(int32Value, m_out.int64), value),
2871             isInt32, isDouble);
2872
2873         LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
2874
2875         results.append(m_out.anchor(boxInt32(int32Value)));
2876         m_out.jump(continuation);
2877
2878         m_out.appendTo(isDouble, continuation);
2879
2880         results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
2881         m_out.jump(continuation);
2882
2883         m_out.appendTo(continuation, lastNext);
2884         return m_out.phi(m_out.int64, results);
2885     }
2886
2887     LValue setInt52WithStrictValue(Node* node, LValue value, Int52Kind kind)
2888     {
2889         switch (kind) {
2890         case StrictInt52:
2891             setStrictInt52(node, value);
2892             return value;
2893
2894         case Int52:
2895             value = strictInt52ToInt52(value);
2896             setInt52(node, value);
2897             return value;
2898         }
2899
2900         RELEASE_ASSERT_NOT_REACHED();
2901         return 0;
2902     }
2903
2904     LValue strictInt52ToInt52(LValue value)
2905     {
2906         return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
2907     }
2908
2909     LValue int52ToStrictInt52(LValue value)
2910     {
2911         return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
2912     }
2913
2914     LValue isNotInt32(LValue jsValue)
2915     {
2916         return m_out.below(jsValue, m_tagTypeNumber);
2917     }
2918     LValue unboxInt32(LValue jsValue)
2919     {
2920         return m_out.castToInt32(jsValue);
2921     }
2922     LValue boxInt32(LValue value)
2923     {
2924         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
2925     }
2926
2927     LValue isCellOrMisc(LValue jsValue)
2928     {
2929         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
2930     }
2931     LValue unboxDouble(LValue jsValue)
2932     {
2933         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
2934     }
2935     LValue boxDouble(LValue doubleValue)
2936     {
2937         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
2938     }
2939
2940     LValue isNotCell(LValue jsValue)
2941     {
2942         return m_out.testNonZero64(jsValue, m_tagMask);
2943     }
2944
2945     LValue isNotBoolean(LValue jsValue)
2946     {
2947         return m_out.testNonZero64(
2948             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
2949             m_out.constInt64(~1));
2950     }
2951     LValue unboxBoolean(LValue jsValue)
2952     {
2953         // We want to use a cast that guarantees that LLVM knows that even the integer
2954         // value is just 0 or 1. But for now we do it the dumb way.
2955         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
2956     }
2957     LValue boxBoolean(LValue value)
2958     {
2959         return m_out.select(
2960             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
2961     }
2962
2963     void speculate(Edge edge)
2964     {
2965         switch (edge.useKind()) {
2966         case UntypedUse:
2967             break;
2968         case KnownInt32Use:
2969         case KnownNumberUse:
2970             ASSERT(!m_interpreter.needsTypeCheck(edge));
2971             break;
2972         case Int32Use:
2973             speculateInt32(edge);
2974             break;
2975         case CellUse:
2976             speculateCell(edge);
2977             break;
2978         case KnownCellUse:
2979             ASSERT(!m_interpreter.needsTypeCheck(edge));
2980             break;
2981         case ObjectUse:
2982             speculateObject(edge);
2983             break;
2984         case ObjectOrOtherUse:
2985             speculateObjectOrOther(edge);
2986             break;
2987         case StringUse:
2988             speculateString(edge);
2989             break;
2990         case RealNumberUse:
2991             speculateRealNumber(edge);
2992             break;
2993         case NumberUse:
2994             speculateNumber(edge);
2995             break;
2996         case BooleanUse:
2997             speculateBoolean(edge);
2998             break;
2999         default:
3000             dataLog("Unsupported speculation use kind: ", edge.useKind(), "\n");
3001             RELEASE_ASSERT_NOT_REACHED();
3002         }
3003     }
3004
3005     void speculate(Node*, Edge edge)
3006     {
3007         speculate(edge);
3008     }
3009
3010     void speculateInt32(Edge edge)
3011     {
3012         lowInt32(edge);
3013     }
3014
3015     void speculateCell(Edge edge)
3016     {
3017         lowCell(edge);
3018     }
3019
3020     LValue isObject(LValue cell)
3021     {
3022         return m_out.notEqual(
3023             m_out.loadPtr(cell, m_heaps.JSCell_structure),
3024             m_out.constIntPtr(vm().stringStructure.get()));
3025     }
3026
3027     LValue isNotString(LValue cell)
3028     {
3029         return isObject(cell);
3030     }
3031
3032     LValue isString(LValue cell)
3033     {
3034         return m_out.equal(
3035             m_out.loadPtr(cell, m_heaps.JSCell_structure),
3036             m_out.constIntPtr(vm().stringStructure.get()));
3037     }
3038
3039     LValue isNotObject(LValue cell)
3040     {
3041         return isString(cell);
3042     }
3043
3044     LValue isArrayType(LValue cell, ArrayMode arrayMode)
3045     {
3046         switch (arrayMode.type()) {
3047         case Array::Int32:
3048         case Array::Double:
3049         case Array::Contiguous: {
3050             LValue indexingType = m_out.load8(
3051                 m_out.loadPtr(cell, m_heaps.JSCell_structure),
3052                 m_heaps.Structure_indexingType);
3053
3054             switch (arrayMode.arrayClass()) {
3055             case Array::OriginalArray:
3056                 RELEASE_ASSERT_NOT_REACHED();
3057                 return 0;
3058
3059             case Array::Array:
3060                 return m_out.equal(
3061                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
3062                     m_out.constInt8(IsArray | arrayMode.shapeMask()));
3063
3064             default:
3065                 return m_out.equal(
3066                     m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
3067                     m_out.constInt8(arrayMode.shapeMask()));
3068             }
3069         }
3070
3071         default:
3072             return hasClassInfo(cell, classInfoForType(arrayMode.typedArrayType()));
3073         }
3074     }
3075
3076     LValue hasClassInfo(LValue cell, const ClassInfo* classInfo)
3077     {
3078         return m_out.equal(
3079             m_out.loadPtr(
3080                 m_out.loadPtr(cell, m_heaps.JSCell_structure),
3081                 m_heaps.Structure_classInfo),
3082             m_out.constIntPtr(classInfo));
3083     }
3084
3085     void speculateObject(Edge edge, LValue cell)
3086     {
3087         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
3088     }
3089
3090     void speculateObject(Edge edge)
3091     {
3092         speculateObject(edge, lowCell(edge));
3093     }
3094
3095     void speculateObjectOrOther(Edge edge)
3096     {
3097         if (!m_interpreter.needsTypeCheck(edge))
3098             return;
3099
3100         LValue value = lowJSValue(edge);
3101
3102         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
3103         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
3104         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
3105
3106         m_out.branch(isNotCell(value), primitiveCase, cellCase);
3107
3108         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
3109
3110         FTL_TYPE_CHECK(
3111             jsValueValue(value), edge, (~SpecCell) | SpecObject,
3112             m_out.equal(
3113                 m_out.loadPtr(value, m_heaps.JSCell_structure),
3114                 m_out.constIntPtr(vm().stringStructure.get())));
3115
3116         m_out.jump(continuation);
3117
3118         m_out.appendTo(primitiveCase, continuation);
3119
3120         FTL_TYPE_CHECK(
3121             jsValueValue(value), edge, SpecCell | SpecOther,
3122             m_out.notEqual(
3123                 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
3124                 m_out.constInt64(ValueNull)));
3125
3126         m_out.jump(continuation);
3127
3128         m_out.appendTo(continuation, lastNext);
3129     }
3130
3131     void speculateString(Edge edge, LValue cell)
3132     {
3133         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
3134     }
3135
3136     void speculateString(Edge edge)
3137     {
3138         speculateString(edge, lowCell(edge));
3139     }
3140
3141     void speculateNonNullObject(Edge edge, LValue cell)
3142     {
3143         LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
3144         FTL_TYPE_CHECK(
3145             jsValueValue(cell), edge, SpecObject,
3146             m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get())));
3147         if (masqueradesAsUndefinedWatchpointIfIsStillValid())
3148             return;
3149
3150         speculate(
3151             BadType, jsValueValue(cell), edge.node(),
3152             m_out.testNonZero8(
3153                 m_out.load8(structure, m_heaps.Structure_typeInfoFlags),
3154                 m_out.constInt8(MasqueradesAsUndefined)));
3155     }
3156
3157     void speculateNumber(Edge edge)
3158     {
3159         // Do an early return here because lowDouble() can create a lot of control flow.
3160         if (!m_interpreter.needsTypeCheck(edge))
3161             return;
3162
3163         lowDouble(edge);
3164     }
3165
3166     void speculateRealNumber(Edge edge)
3167     {
3168         // Do an early return here because lowDouble() can create a lot of control flow.
3169         if (!m_interpreter.needsTypeCheck(edge))
3170             return;
3171
3172         LValue value = lowDouble(edge);
3173         FTL_TYPE_CHECK(
3174             doubleValue(value), edge, SpecFullRealNumber,
3175             m_out.doubleNotEqualOrUnordered(value, value));
3176     }
3177
3178     void speculateBoolean(Edge edge)
3179     {
3180         lowBoolean(edge);
3181     }
3182
3183     bool masqueradesAsUndefinedWatchpointIsStillValid()
3184     {
3185         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin);
3186     }
3187
3188     bool masqueradesAsUndefinedWatchpointIfIsStillValid()
3189     {
3190         if (!masqueradesAsUndefinedWatchpointIsStillValid())
3191             return false;
3192
3193         // FIXME: Implement masquerades-as-undefined watchpoints.
3194         // https://bugs.webkit.org/show_bug.cgi?id=113647
3195         return true;
3196     }
3197
3198     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
3199
3200     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
3201     {
3202         callPreflight();
3203         LValue result = m_out.call(function);
3204         callCheck(mode);
3205         return result;
3206     }
3207     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
3208     {
3209         callPreflight();
3210         LValue result = m_out.call(function, arg1);
3211         callCheck(mode);
3212         return result;
3213     }
3214     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
3215     {
3216         callPreflight();
3217         LValue result = m_out.call(function, arg1, arg2);
3218         callCheck(mode);
3219         return result;
3220     }
3221     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
3222     {
3223         callPreflight();
3224         LValue result = m_out.call(function, arg1, arg2, arg3);
3225         callCheck(mode);
3226         return result;
3227     }
3228     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
3229     {
3230         callPreflight();
3231         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
3232         callCheck(mode);
3233         return result;
3234     }
3235
3236     void callPreflight(CodeOrigin codeOrigin)
3237     {
3238         m_out.store32(
3239             m_out.constInt32(
3240                 CallFrame::Location::encodeAsCodeOriginIndex(
3241                     m_ftlState.jitCode->common.addCodeOrigin(codeOrigin))),
3242             tagFor(JSStack::ArgumentCount));
3243     }
3244     void callPreflight()
3245     {
3246         callPreflight(m_node->codeOrigin);
3247     }
3248
3249     void callCheck(ExceptionCheckMode mode = CheckExceptions)
3250     {
3251         if (mode == NoExceptions)
3252             return;
3253
3254         LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception"));
3255         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
3256
3257         m_out.branch(
3258             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
3259             didHaveException, continuation);
3260
3261         LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation);
3262         // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622
3263         m_out.crash();
3264
3265         m_out.appendTo(continuation, lastNext);
3266     }
3267
3268     bool isLive(Node* node)
3269     {
3270         return m_live.contains(node);
3271     }
3272
3273     void use(Edge edge)
3274     {
3275         ASSERT(edge->hasResult());
3276         if (!edge.doesKill())
3277             return;
3278         m_live.remove(edge.node());
3279     }
3280
3281     // Wrapper used only for DFG_NODE_DO_TO_CHILDREN
3282     void use(Node*, Edge edge)
3283     {
3284         use(edge);
3285     }
3286
3287     LBasicBlock lowBlock(BasicBlock* block)
3288     {
3289         return m_blocks.get(block);
3290     }
3291
3292     void initializeOSRExitStateForBlock()
3293     {
3294         for (unsigned i = m_valueSources.size(); i--;) {
3295             FlushFormat format = m_highBlock->ssa->flushFormatAtHead[i];
3296             switch (format) {
3297             case DeadFlush: {
3298                 // Must consider available nodes instead.
3299                 Node* node = m_highBlock->ssa->availabilityAtHead[i];
3300                 if (!node) {
3301                     m_valueSources[i] = ValueSource(SourceIsDead);
3302                     break;
3303                 }
3304
3305                 m_valueSources[i] = ValueSource(node);
3306                 break;
3307             }
3308
3309             case FlushedInt32:
3310                 m_valueSources[i] = ValueSource(Int32InJSStack);
3311                 break;
3312
3313             case FlushedInt52:
3314                 m_valueSources[i] = ValueSource(Int52InJSStack);
3315                 break;
3316
3317             case FlushedDouble:
3318                 m_valueSources[i] = ValueSource(DoubleInJSStack);
3319                 break;
3320
3321             case FlushedCell:
3322             case FlushedBoolean:
3323             case FlushedJSValue:
3324                 m_valueSources[i] = ValueSource(ValueInJSStack);
3325                 break;
3326             }
3327         }
3328     }
3329
3330     void appendOSRExit(
3331         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
3332         SpeculationDirection direction, FormattedValue recovery)
3333     {
3334         if (Options::ftlTrapsOnOSRExit()) {
3335             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
3336             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
3337
3338             m_out.branch(failCondition, failCase, continuation);
3339
3340             LBasicBlock lastNext = m_out.appendTo(failCase, continuation);
3341             m_out.trap();
3342             m_out.unreachable();
3343
3344             m_out.appendTo(continuation, lastNext);
3345             return;
3346         }
3347
3348         if (verboseCompilationEnabled())
3349             dataLog("    OSR exit with value sources: ", m_valueSources, "\n");
3350
3351         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.osrExit.size());
3352         unsigned index = m_ftlState.osrExit.size();
3353
3354         m_ftlState.jitCode->osrExit.append(OSRExit(
3355             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
3356             m_codeOriginForExitTarget, m_codeOriginForExitProfile, m_lastSetOperand.offset(),
3357             m_valueSources.numberOfArguments(), m_valueSources.numberOfLocals()));
3358         m_ftlState.osrExit.append(OSRExitCompilationInfo());
3359
3360         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
3361         OSRExitCompilationInfo& info = m_ftlState.osrExit.last();
3362
3363         LBasicBlock lastNext = 0;
3364         LBasicBlock continuation = 0;
3365
3366         if (!Options::useLLVMOSRExitIntrinsic()) {
3367             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
3368             continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
3369
3370             m_out.branch(failCondition, failCase, continuation);
3371
3372             m_out.appendTo(m_prologue);
3373             info.m_thunkAddress = buildAlloca(m_out.m_builder, m_out.intPtr);
3374
3375             lastNext = m_out.appendTo(failCase, continuation);
3376         }
3377
3378         if (Options::ftlOSRExitOmitsMarshalling()) {
3379             m_out.call(
3380                 m_out.intToPtr(
3381                     m_out.get(info.m_thunkAddress),
3382                     pointerType(functionType(m_out.voidType))));
3383         } else
3384             emitOSRExitCall(failCondition, index, exit, info, lowValue, direction, recovery);
3385
3386         if (!Options::useLLVMOSRExitIntrinsic()) {
3387             m_out.unreachable();
3388
3389             m_out.appendTo(continuation, lastNext);
3390
3391             m_exitThunkGenerator.emitThunk(index);
3392         }
3393     }
3394
3395     void emitOSRExitCall(
3396         LValue failCondition, unsigned index, OSRExit& exit, OSRExitCompilationInfo& info,
3397         FormattedValue lowValue, SpeculationDirection direction, FormattedValue recovery)
3398     {
3399         ExitArgumentList arguments;
3400
3401         if (Options::useLLVMOSRExitIntrinsic()) {
3402             arguments.append(failCondition);
3403             arguments.append(m_out.constInt32(index));
3404         }
3405
3406         arguments.append(m_callFrame);
3407         if (!!lowValue)
3408             arguments.append(lowValue.value());
3409
3410         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
3411             ValueSource source = m_valueSources[i];
3412
3413             switch (source.kind()) {
3414             case ValueInJSStack:
3415                 exit.m_values[i] = ExitValue::inJSStack();
3416                 break;
3417             case Int32InJSStack:
3418                 exit.m_values[i] = ExitValue::inJSStackAsInt32();
3419                 break;
3420             case Int52InJSStack:
3421                 exit.m_values[i] = ExitValue::inJSStackAsInt52();
3422                 break;
3423             case DoubleInJSStack:
3424                 exit.m_values[i] = ExitValue::inJSStackAsDouble();
3425                 break;
3426             case SourceIsDead:
3427                 exit.m_values[i] = ExitValue::dead();
3428                 break;
3429             case HaveNode:
3430                 addExitArgumentForNode(exit, arguments, i, source.node());
3431                 break;
3432             default:
3433                 RELEASE_ASSERT_NOT_REACHED();
3434                 break;
3435             }
3436         }
3437
3438         if (verboseCompilationEnabled())
3439             dataLog("        Exit values: ", exit.m_values, "\n");
3440
3441         if (direction == ForwardSpeculation) {
3442             ASSERT(m_node);
3443             exit.convertToForward(m_highBlock, m_node, m_nodeIndex, recovery, arguments);
3444         }
3445
3446         // So, the really lame thing here is that we have to build an LLVM function type.
3447         // Booo.
3448         Vector<LType, 16> argumentTypes;
3449         for (unsigned i = 0; i < arguments.size(); ++i)
3450             argumentTypes.append(typeOf(arguments[i]));
3451
3452         if (Options::useLLVMOSRExitIntrinsic()) {
3453             m_out.call(m_out.osrExitIntrinsic(), arguments);
3454             return;
3455         }
3456
3457         m_out.call(
3458             m_out.intToPtr(
3459                 m_out.get(info.m_thunkAddress),
3460                 pointerType(functionType(m_out.voidType, argumentTypes))),
3461             arguments);
3462     }
3463
3464     void addExitArgumentForNode(
3465         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
3466     {
3467         ASSERT(node->shouldGenerate());
3468         ASSERT(node->hasResult());
3469
3470         if (tryToSetConstantExitArgument(exit, index, node))
3471             return;
3472
3473         if (!isLive(node)) {
3474             bool found = false;
3475
3476             if (permitsOSRBackwardRewiring(node->op())) {
3477                 node = node->child1().node();
3478                 if (tryToSetConstantExitArgument(exit, index, node))
3479                     return;
3480                 if (isLive(node))
3481                     found = true;
3482             }
3483
3484             if (!found) {
3485                 Node* bestNode = 0;
3486                 unsigned bestScore = 0;
3487
3488                 HashSet<Node*>::iterator iter = m_live.begin();
3489                 HashSet<Node*>::iterator end = m_live.end();
3490                 for (; iter != end; ++iter) {
3491                     Node* candidate = *iter;
3492                     if (candidate->flags() & NodeHasVarArgs)
3493                         continue;
3494                     if (!candidate->child1())
3495                         continue;
3496                     if (candidate->child1() != node)
3497                         continue;
3498                     unsigned myScore = forwardRewiringSelectionScore(candidate->op());
3499                     if (myScore <= bestScore)
3500                         continue;
3501                     bestNode = candidate;
3502                     bestScore = myScore;
3503                 }
3504
3505                 if (bestNode) {
3506                     ASSERT(isLive(bestNode));
3507                     node = bestNode;
3508                     found = true;
3509                 }
3510             }
3511
3512             if (!found) {
3513                 exit.m_values[index] = ExitValue::dead();
3514                 return;
3515             }
3516         }
3517
3518         ASSERT(isLive(node));
3519
3520         LoweredNodeValue value = m_int32Values.get(node);
3521         if (isValid(value)) {
3522             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
3523             return;
3524         }
3525
3526         value = m_int52Values.get(node);
3527         if (isValid(value)) {
3528             addExitArgument(exit, arguments, index, ValueFormatInt52, value.value());
3529             return;
3530         }
3531
3532         value = m_strictInt52Values.get(node);
3533         if (isValid(value)) {
3534             addExitArgument(exit, arguments, index, ValueFormatStrictInt52, value.value());
3535             return;
3536         }
3537
3538         value = m_booleanValues.get(node);
3539         if (isValid(value)) {
3540             addExitArgument(exit, arguments, index, ValueFormatBoolean, value.value());
3541             return;
3542         }
3543
3544         value = m_jsValueValues.get(node);
3545         if (isValid(value)) {
3546             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
3547             return;
3548         }
3549
3550         value = m_doubleValues.get(node);
3551         if (isValid(value)) {
3552             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
3553             return;
3554         }
3555
3556         dataLog("Cannot find value for node: ", node, "\n");
3557         RELEASE_ASSERT_NOT_REACHED();
3558     }
3559
3560     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
3561     {
3562         if (!node)
3563             return false;
3564
3565         switch (node->op()) {
3566         case JSConstant:
3567         case WeakJSConstant:
3568             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
3569             return true;
3570         case PhantomArguments:
3571             // FIXME: implement PhantomArguments.
3572             // https://bugs.webkit.org/show_bug.cgi?id=113986
3573             RELEASE_ASSERT_NOT_REACHED();
3574             return true;
3575         default:
3576             return false;
3577         }
3578     }
3579
3580     void addExitArgument(
3581         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
3582         LValue value)
3583     {
3584         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
3585         arguments.append(value);
3586     }
3587
3588     void linkOSRExitsAndCompleteInitializationBlocks()
3589     {
3590         MacroAssemblerCodeRef osrExitThunk =
3591             vm().getCTIStub(osrExitGenerationThunkGenerator);
3592         CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
3593
3594         m_out.appendTo(m_prologue);
3595         m_out.jump(m_initialization);
3596
3597         m_out.appendTo(m_initialization);
3598
3599         if (m_exitThunkGenerator.didThings()) {
3600             OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
3601                 vm(), &m_exitThunkGenerator, m_ftlState.graph.m_codeBlock,
3602                 JITCompilationMustSucceed));
3603
3604             ASSERT(m_ftlState.osrExit.size() == m_ftlState.jitCode->osrExit.size());
3605
3606             for (unsigned i = 0; i < m_ftlState.osrExit.size(); ++i) {
3607                 OSRExitCompilationInfo& info = m_ftlState.osrExit[i];
3608                 OSRExit& exit = m_ftlState.jitCode->osrExit[i];
3609
3610                 linkBuffer->link(info.m_thunkJump, target);
3611
3612           &n