Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "DFGOSRAvailabilityAnalysisPhase.h"
  35 #include "DirectArguments.h"
  36 #include "FTLAbstractHeapRepository.h"
  37 #include "FTLAvailableRecovery.h"
  38 #include "FTLForOSREntryJITCode.h"
  39 #include "FTLFormattedValue.h"
  40 #include "FTLInlineCacheSize.h"
  41 #include "FTLLoweredNodeValue.h"
  42 #include "FTLOperations.h"
  43 #include "FTLOutput.h"
  44 #include "FTLThunks.h"
  45 #include "FTLWeightedTarget.h"
  46 #include "JSCInlines.h"
  47 #include "JSLexicalEnvironment.h"
  48 #include "OperandsInlines.h"
  49 #include "ScopedArguments.h"
  50 #include "ScopedArgumentsTable.h"
  51 #include "VirtualRegister.h"
  52 #include <atomic>
  53 #include <dlfcn.h>
  54 #include <llvm/InitializeLLVM.h>
  55 #include <unordered_set>
  56 #include <wtf/ProcessID.h>
  57
  58 #if ENABLE(FTL_NATIVE_CALL_INLINING)
  59 #include "BundlePath.h"
  60 #endif
  61
  62 namespace JSC { namespace FTL {
  63
  64 using namespace DFG;
  65
  66 static std::atomic<int> compileCounter;
  67
  68 #if ASSERT_DISABLED
  69 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
  70 {
  71     CRASH();
  72 }
  73 #else
  74 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
  75     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
  76 {
  77     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
  78     if (nodeIndex != UINT_MAX)
  79         dataLog(", node @", nodeIndex);
  80     dataLog(".\n");
  81     CRASH();
  82 }
  83 #endif
  84
  85 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  86 // significantly less dead code.
  87 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  88         FormattedValue _ftc_lowValue = (lowValue);                      \
  89         Edge _ftc_highValue = (highValue);                              \
  90         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  91         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  92             break;                                                      \
  93         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  94     } while (false)
  95
  96 class LowerDFGToLLVM {
  97 public:
  98     LowerDFGToLLVM(State& state)
  99         : m_graph(state.graph)
 100         , m_ftlState(state)
 101         , m_heaps(state.context)
 102         , m_out(state.context)
 103         , m_state(state.graph)
 104         , m_interpreter(state.graph, m_state)
 105         , m_stackmapIDs(0)
 106         , m_tbaaKind(mdKindID(state.context, "tbaa"))
 107         , m_tbaaStructKind(mdKindID(state.context, "tbaa.struct"))
 108     {
 109     }
 110
 111     void lower()
 112     {
 113         CString name;
 114         if (verboseCompilationEnabled()) {
 115             name = toCString(
 116                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
 117                 "_", codeBlock()->hash());
 118         } else
 119             name = "jsBody";
 120
 121         m_graph.m_dominators.computeIfNecessary(m_graph);
 122
 123         m_ftlState.module =
 124             moduleCreateWithNameInContext(name.data(), m_ftlState.context);
 125
 126         m_ftlState.function = addFunction(
 127             m_ftlState.module, name.data(), functionType(m_out.int64));
 128         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
 129         if (isX86() && Options::llvmDisallowAVX()) {
 130             // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
 131             // slower. It should be disabled.
 132             addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
 133         }
 134
 135         if (verboseCompilationEnabled())
 136             dataLog("Function ready, beginning lowering.\n");
 137
 138         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 139
 140         m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
 141         LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
 142         m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
 143
 144         LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments"));
 145
 146         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 147             m_highBlock = m_graph.block(blockIndex);
 148             if (!m_highBlock)
 149                 continue;
 150             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 151         }
 152
 153         m_out.appendTo(m_prologue, stackOverflow);
 154         createPhiVariables();
 155
 156         auto preOrder = m_graph.blocksInPreOrder();
 157
 158         int maxNumberOfArguments = -1;
 159         for (BasicBlock* block : preOrder) {
 160             for (unsigned nodeIndex = block->size(); nodeIndex--; ) {
 161                 Node* node = block->at(nodeIndex);
 162                 switch (node->op()) {
 163                 case NativeCall:
 164                 case NativeConstruct: {
 165                     int numArgs = node->numChildren();
 166                     if (numArgs > maxNumberOfArguments)
 167                         maxNumberOfArguments = numArgs;
 168                     break;
 169                 }
 170                 default:
 171                     break;
 172                 }
 173             }
 174         }
 175
 176         if (maxNumberOfArguments >= 0) {
 177             m_execState = m_out.alloca(arrayType(m_out.int64, JSStack::CallFrameHeaderSize + maxNumberOfArguments));
 178             m_execStorage = m_out.ptrToInt(m_execState, m_out.intPtr);
 179         }
 180
 181         LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
 182
 183         m_captured = m_out.add(
 184             m_out.ptrToInt(capturedAlloca, m_out.intPtr),
 185             m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
 186
 187         m_ftlState.capturedStackmapID = m_stackmapIDs++;
 188         m_out.call(
 189             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
 190             m_out.int32Zero, capturedAlloca);
 191
 192         // If we have any CallVarargs then we nee to have a spill slot for it.
 193         bool hasVarargs = false;
 194         for (BasicBlock* block : preOrder) {
 195             for (Node* node : *block) {
 196                 switch (node->op()) {
 197                 case CallVarargs:
 198                 case CallForwardVarargs:
 199                 case ConstructVarargs:
 200                 case ConstructForwardVarargs:
 201                     hasVarargs = true;
 202                     break;
 203                 default:
 204                     break;
 205                 }
 206             }
 207         }
 208         if (hasVarargs) {
 209             LValue varargsSpillSlots = m_out.alloca(
 210                 arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded()));
 211             m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++;
 212             m_out.call(
 213                 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID),
 214                 m_out.int32Zero, varargsSpillSlots);
 215         }
 216
 217         // We should not create any alloca's after this point, since they will cease to
 218         // be mem2reg candidates.
 219
 220         m_callFrame = m_out.ptrToInt(
 221             m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
 222         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 223         m_tagMask = m_out.constInt64(TagMask);
 224
 225         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 226
 227         m_out.branch(
 228             didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
 229
 230         m_out.appendTo(stackOverflow, m_handleExceptions);
 231         m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 232         m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
 233         m_out.call(
 234             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
 235             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 236         m_out.unreachable();
 237
 238         m_out.appendTo(m_handleExceptions, checkArguments);
 239         m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
 240         m_out.call(
 241             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
 242             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 243         m_out.unreachable();
 244
 245         m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
 246         availabilityMap().clear();
 247         availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
 248         for (unsigned i = codeBlock()->numParameters(); i--;) {
 249             availabilityMap().m_locals.argument(i) =
 250                 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
 251         }
 252         m_codeOriginForExitTarget = CodeOrigin(0);
 253         m_codeOriginForExitProfile = CodeOrigin(0);
 254         m_node = nullptr;
 255         for (unsigned i = codeBlock()->numParameters(); i--;) {
 256             Node* node = m_graph.m_arguments[i];
 257             VirtualRegister operand = virtualRegisterForArgument(i);
 258
 259             LValue jsValue = m_out.load64(addressFor(operand));
 260
 261             if (node) {
 262                 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
 263
 264                 // This is a hack, but it's an effective one. It allows us to do CSE on the
 265                 // primordial load of arguments. This assumes that the GetLocal that got put in
 266                 // place of the original SetArgument doesn't have any effects before it. This
 267                 // should hold true.
 268                 m_loadedArgumentValues.add(node, jsValue);
 269             }
 270
 271             switch (m_graph.m_argumentFormats[i]) {
 272             case FlushedInt32:
 273                 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
 274                 break;
 275             case FlushedBoolean:
 276                 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
 277                 break;
 278             case FlushedCell:
 279                 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
 280                 break;
 281             case FlushedJSValue:
 282                 break;
 283             default:
 284                 DFG_CRASH(m_graph, node, "Bad flush format for argument");
 285                 break;
 286             }
 287         }
 288         m_out.jump(lowBlock(m_graph.block(0)));
 289
 290         for (BasicBlock* block : preOrder)
 291             compileBlock(block);
 292
 293         if (Options::dumpLLVMIR())
 294             dumpModule(m_ftlState.module);
 295
 296         if (verboseCompilationEnabled())
 297             m_ftlState.dumpState("after lowering");
 298         if (validationEnabled())
 299             verifyModule(m_ftlState.module);
 300     }
 301
 302 private:
 303
 304     void createPhiVariables()
 305     {
 306         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 307             BasicBlock* block = m_graph.block(blockIndex);
 308             if (!block)
 309                 continue;
 310             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 311                 Node* node = block->at(nodeIndex);
 312                 if (node->op() != Phi)
 313                     continue;
 314                 LType type;
 315                 switch (node->flags() & NodeResultMask) {
 316                 case NodeResultDouble:
 317                     type = m_out.doubleType;
 318                     break;
 319                 case NodeResultInt32:
 320                     type = m_out.int32;
 321                     break;
 322                 case NodeResultInt52:
 323                     type = m_out.int64;
 324                     break;
 325                 case NodeResultBoolean:
 326                     type = m_out.boolean;
 327                     break;
 328                 case NodeResultJS:
 329                     type = m_out.int64;
 330                     break;
 331                 default:
 332                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
 333                     break;
 334                 }
 335                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 336             }
 337         }
 338     }
 339
 340     void compileBlock(BasicBlock* block)
 341     {
 342         if (!block)
 343             return;
 344
 345         if (verboseCompilationEnabled())
 346             dataLog("Compiling block ", *block, "\n");
 347
 348         m_highBlock = block;
 349
 350         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 351
 352         m_nextHighBlock = 0;
 353         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 354             m_nextHighBlock = m_graph.block(nextBlockIndex);
 355             if (m_nextHighBlock)
 356                 break;
 357         }
 358         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 359
 360         // All of this effort to find the next block gives us the ability to keep the
 361         // generated IR in roughly program order. This ought not affect the performance
 362         // of the generated code (since we expect LLVM to reorder things) but it will
 363         // make IR dumps easier to read.
 364         m_out.appendTo(lowBlock, m_nextLowBlock);
 365
 366         if (Options::ftlCrashes())
 367             m_out.trap();
 368
 369         if (!m_highBlock->cfaHasVisited) {
 370             if (verboseCompilationEnabled())
 371                 dataLog("Bailing because CFA didn't reach.\n");
 372             crash(m_highBlock->index, UINT_MAX);
 373             return;
 374         }
 375
 376         m_availabilityCalculator.beginBlock(m_highBlock);
 377
 378         m_state.reset();
 379         m_state.beginBasicBlock(m_highBlock);
 380
 381         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 382             if (!compileNode(m_nodeIndex))
 383                 break;
 384         }
 385     }
 386
 387     void safelyInvalidateAfterTermination()
 388     {
 389         if (verboseCompilationEnabled())
 390             dataLog("Bailing.\n");
 391         crash();
 392
 393         // Invalidate dominated blocks. Under normal circumstances we would expect
 394         // them to be invalidated already. But you can have the CFA become more
 395         // precise over time because the structures of objects change on the main
 396         // thread. Failing to do this would result in weird crashes due to a value
 397         // being used but not defined. Race conditions FTW!
 398         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 399             BasicBlock* target = m_graph.block(blockIndex);
 400             if (!target)
 401                 continue;
 402             if (m_graph.m_dominators.dominates(m_highBlock, target)) {
 403                 if (verboseCompilationEnabled())
 404                     dataLog("Block ", *target, " will bail also.\n");
 405                 target->cfaHasVisited = false;
 406             }
 407         }
 408     }
 409
 410     bool compileNode(unsigned nodeIndex)
 411     {
 412         if (!m_state.isValid()) {
 413             safelyInvalidateAfterTermination();
 414             return false;
 415         }
 416
 417         m_node = m_highBlock->at(nodeIndex);
 418         m_codeOriginForExitProfile = m_node->origin.semantic;
 419         m_codeOriginForExitTarget = m_node->origin.forExit;
 420
 421         if (verboseCompilationEnabled())
 422             dataLog("Lowering ", m_node, "\n");
 423
 424         m_availableRecoveries.resize(0);
 425
 426         m_interpreter.startExecuting();
 427
 428         switch (m_node->op()) {
 429         case Upsilon:
 430             compileUpsilon();
 431             break;
 432         case Phi:
 433             compilePhi();
 434             break;
 435         case JSConstant:
 436             break;
 437         case DoubleConstant:
 438             compileDoubleConstant();
 439             break;
 440         case Int52Constant:
 441             compileInt52Constant();
 442             break;
 443         case DoubleRep:
 444             compileDoubleRep();
 445             break;
 446         case DoubleAsInt32:
 447             compileDoubleAsInt32();
 448             break;
 449         case ValueRep:
 450             compileValueRep();
 451             break;
 452         case Int52Rep:
 453             compileInt52Rep();
 454             break;
 455         case ValueToInt32:
 456             compileValueToInt32();
 457             break;
 458         case BooleanToNumber:
 459             compileBooleanToNumber();
 460             break;
 461         case ExtractOSREntryLocal:
 462             compileExtractOSREntryLocal();
 463             break;
 464         case GetStack:
 465             compileGetStack();
 466             break;
 467         case PutStack:
 468             compilePutStack();
 469             break;
 470         case Check:
 471             compileNoOp();
 472             break;
 473         case ToThis:
 474             compileToThis();
 475             break;
 476         case ValueAdd:
 477             compileValueAdd();
 478             break;
 479         case ArithAdd:
 480         case ArithSub:
 481             compileArithAddOrSub();
 482             break;
 483         case ArithClz32:
 484             compileArithClz32();
 485             break;
 486         case ArithMul:
 487             compileArithMul();
 488             break;
 489         case ArithDiv:
 490             compileArithDiv();
 491             break;
 492         case ArithMod:
 493             compileArithMod();
 494             break;
 495         case ArithMin:
 496         case ArithMax:
 497             compileArithMinOrMax();
 498             break;
 499         case ArithAbs:
 500             compileArithAbs();
 501             break;
 502         case ArithSin:
 503             compileArithSin();
 504             break;
 505         case ArithCos:
 506             compileArithCos();
 507             break;
 508         case ArithPow:
 509             compileArithPow();
 510             break;
 511         case ArithRound:
 512             compileArithRound();
 513             break;
 514         case ArithSqrt:
 515             compileArithSqrt();
 516             break;
 517         case ArithLog:
 518             compileArithLog();
 519             break;
 520         case ArithFRound:
 521             compileArithFRound();
 522             break;
 523         case ArithNegate:
 524             compileArithNegate();
 525             break;
 526         case BitAnd:
 527             compileBitAnd();
 528             break;
 529         case BitOr:
 530             compileBitOr();
 531             break;
 532         case BitXor:
 533             compileBitXor();
 534             break;
 535         case BitRShift:
 536             compileBitRShift();
 537             break;
 538         case BitLShift:
 539             compileBitLShift();
 540             break;
 541         case BitURShift:
 542             compileBitURShift();
 543             break;
 544         case UInt32ToNumber:
 545             compileUInt32ToNumber();
 546             break;
 547         case CheckStructure:
 548             compileCheckStructure();
 549             break;
 550         case CheckCell:
 551             compileCheckCell();
 552             break;
 553         case CheckNotEmpty:
 554             compileCheckNotEmpty();
 555             break;
 556         case CheckBadCell:
 557             compileCheckBadCell();
 558             break;
 559         case GetExecutable:
 560             compileGetExecutable();
 561             break;
 562         case ArrayifyToStructure:
 563             compileArrayifyToStructure();
 564             break;
 565         case PutStructure:
 566             compilePutStructure();
 567             break;
 568         case GetById:
 569             compileGetById();
 570             break;
 571         case In:
 572             compileIn();
 573             break;
 574         case PutById:
 575         case PutByIdDirect:
 576             compilePutById();
 577             break;
 578         case GetButterfly:
 579             compileGetButterfly();
 580             break;
 581         case ConstantStoragePointer:
 582             compileConstantStoragePointer();
 583             break;
 584         case GetIndexedPropertyStorage:
 585             compileGetIndexedPropertyStorage();
 586             break;
 587         case CheckArray:
 588             compileCheckArray();
 589             break;
 590         case GetArrayLength:
 591             compileGetArrayLength();
 592             break;
 593         case CheckInBounds:
 594             compileCheckInBounds();
 595             break;
 596         case GetByVal:
 597             compileGetByVal();
 598             break;
 599         case GetMyArgumentByVal:
 600             compileGetMyArgumentByVal();
 601             break;
 602         case PutByVal:
 603         case PutByValAlias:
 604         case PutByValDirect:
 605             compilePutByVal();
 606             break;
 607         case ArrayPush:
 608             compileArrayPush();
 609             break;
 610         case ArrayPop:
 611             compileArrayPop();
 612             break;
 613         case CreateActivation:
 614             compileCreateActivation();
 615             break;
 616         case NewFunction:
 617             compileNewFunction();
 618             break;
 619         case CreateDirectArguments:
 620             compileCreateDirectArguments();
 621             break;
 622         case CreateScopedArguments:
 623             compileCreateScopedArguments();
 624             break;
 625         case CreateClonedArguments:
 626             compileCreateClonedArguments();
 627             break;
 628         case NewObject:
 629             compileNewObject();
 630             break;
 631         case NewArray:
 632             compileNewArray();
 633             break;
 634         case NewArrayBuffer:
 635             compileNewArrayBuffer();
 636             break;
 637         case NewArrayWithSize:
 638             compileNewArrayWithSize();
 639             break;
 640         case GetTypedArrayByteOffset:
 641             compileGetTypedArrayByteOffset();
 642             break;
 643         case AllocatePropertyStorage:
 644             compileAllocatePropertyStorage();
 645             break;
 646         case ReallocatePropertyStorage:
 647             compileReallocatePropertyStorage();
 648             break;
 649         case ToString:
 650         case CallStringConstructor:
 651             compileToStringOrCallStringConstructor();
 652             break;
 653         case ToPrimitive:
 654             compileToPrimitive();
 655             break;
 656         case MakeRope:
 657             compileMakeRope();
 658             break;
 659         case StringCharAt:
 660             compileStringCharAt();
 661             break;
 662         case StringCharCodeAt:
 663             compileStringCharCodeAt();
 664             break;
 665         case GetByOffset:
 666         case GetGetterSetterByOffset:
 667             compileGetByOffset();
 668             break;
 669         case GetGetter:
 670             compileGetGetter();
 671             break;
 672         case GetSetter:
 673             compileGetSetter();
 674             break;
 675         case MultiGetByOffset:
 676             compileMultiGetByOffset();
 677             break;
 678         case PutByOffset:
 679             compilePutByOffset();
 680             break;
 681         case MultiPutByOffset:
 682             compileMultiPutByOffset();
 683             break;
 684         case GetGlobalVar:
 685             compileGetGlobalVar();
 686             break;
 687         case PutGlobalVar:
 688             compilePutGlobalVar();
 689             break;
 690         case NotifyWrite:
 691             compileNotifyWrite();
 692             break;
 693         case GetCallee:
 694             compileGetCallee();
 695             break;
 696         case GetArgumentCount:
 697             compileGetArgumentCount();
 698             break;
 699         case GetScope:
 700             compileGetScope();
 701             break;
 702         case SkipScope:
 703             compileSkipScope();
 704             break;
 705         case GetClosureVar:
 706             compileGetClosureVar();
 707             break;
 708         case PutClosureVar:
 709             compilePutClosureVar();
 710             break;
 711         case GetFromArguments:
 712             compileGetFromArguments();
 713             break;
 714         case PutToArguments:
 715             compilePutToArguments();
 716             break;
 717         case CompareEq:
 718             compileCompareEq();
 719             break;
 720         case CompareEqConstant:
 721             compileCompareEqConstant();
 722             break;
 723         case CompareStrictEq:
 724             compileCompareStrictEq();
 725             break;
 726         case CompareLess:
 727             compileCompareLess();
 728             break;
 729         case CompareLessEq:
 730             compileCompareLessEq();
 731             break;
 732         case CompareGreater:
 733             compileCompareGreater();
 734             break;
 735         case CompareGreaterEq:
 736             compileCompareGreaterEq();
 737             break;
 738         case LogicalNot:
 739             compileLogicalNot();
 740             break;
 741         case Call:
 742         case Construct:
 743             compileCallOrConstruct();
 744             break;
 745         case CallVarargs:
 746         case CallForwardVarargs:
 747         case ConstructVarargs:
 748         case ConstructForwardVarargs:
 749             compileCallOrConstructVarargs();
 750             break;
 751         case LoadVarargs:
 752             compileLoadVarargs();
 753             break;
 754         case ForwardVarargs:
 755             compileForwardVarargs();
 756             break;
 757 #if ENABLE(FTL_NATIVE_CALL_INLINING)
 758         case NativeCall:
 759         case NativeConstruct:
 760             compileNativeCallOrConstruct();
 761             break;
 762 #endif
 763         case Jump:
 764             compileJump();
 765             break;
 766         case Branch:
 767             compileBranch();
 768             break;
 769         case Switch:
 770             compileSwitch();
 771             break;
 772         case Return:
 773             compileReturn();
 774             break;
 775         case ForceOSRExit:
 776             compileForceOSRExit();
 777             break;
 778         case Throw:
 779         case ThrowReferenceError:
 780             compileThrow();
 781             break;
 782         case InvalidationPoint:
 783             compileInvalidationPoint();
 784             break;
 785         case IsUndefined:
 786             compileIsUndefined();
 787             break;
 788         case IsBoolean:
 789             compileIsBoolean();
 790             break;
 791         case IsNumber:
 792             compileIsNumber();
 793             break;
 794         case IsString:
 795             compileIsString();
 796             break;
 797         case IsObject:
 798             compileIsObject();
 799             break;
 800         case IsObjectOrNull:
 801             compileIsObjectOrNull();
 802             break;
 803         case IsFunction:
 804             compileIsFunction();
 805             break;
 806         case TypeOf:
 807             compileTypeOf();
 808             break;
 809         case CheckHasInstance:
 810             compileCheckHasInstance();
 811             break;
 812         case InstanceOf:
 813             compileInstanceOf();
 814             break;
 815         case CountExecution:
 816             compileCountExecution();
 817             break;
 818         case StoreBarrier:
 819             compileStoreBarrier();
 820             break;
 821         case HasIndexedProperty:
 822             compileHasIndexedProperty();
 823             break;
 824         case HasGenericProperty:
 825             compileHasGenericProperty();
 826             break;
 827         case HasStructureProperty:
 828             compileHasStructureProperty();
 829             break;
 830         case GetDirectPname:
 831             compileGetDirectPname();
 832             break;
 833         case GetEnumerableLength:
 834             compileGetEnumerableLength();
 835             break;
 836         case GetPropertyEnumerator:
 837             compileGetPropertyEnumerator();
 838             break;
 839         case GetEnumeratorStructurePname:
 840             compileGetEnumeratorStructurePname();
 841             break;
 842         case GetEnumeratorGenericPname:
 843             compileGetEnumeratorGenericPname();
 844             break;
 845         case ToIndexString:
 846             compileToIndexString();
 847             break;
 848         case CheckStructureImmediate:
 849             compileCheckStructureImmediate();
 850             break;
 851         case MaterializeNewObject:
 852             compileMaterializeNewObject();
 853             break;
 854         case MaterializeCreateActivation:
 855             compileMaterializeCreateActivation();
 856             break;
 857
 858         case PhantomLocal:
 859         case LoopHint:
 860         case MovHint:
 861         case ZombieHint:
 862         case PhantomNewObject:
 863         case PhantomNewFunction:
 864         case PhantomCreateActivation:
 865         case PhantomDirectArguments:
 866         case PhantomClonedArguments:
 867         case PutHint:
 868         case BottomValue:
 869         case KillStack:
 870             break;
 871         default:
 872             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
 873             break;
 874         }
 875
 876         if (m_node->isTerminal())
 877             return false;
 878
 879         if (!m_state.isValid()) {
 880             safelyInvalidateAfterTermination();
 881             return false;
 882         }
 883
 884         m_availabilityCalculator.executeNode(m_node);
 885         m_interpreter.executeEffects(nodeIndex);
 886
 887         return true;
 888     }
 889
 890     void compileUpsilon()
 891     {
 892         LValue destination = m_phis.get(m_node->phi());
 893
 894         switch (m_node->child1().useKind()) {
 895         case DoubleRepUse:
 896             m_out.set(lowDouble(m_node->child1()), destination);
 897             break;
 898         case Int32Use:
 899             m_out.set(lowInt32(m_node->child1()), destination);
 900             break;
 901         case Int52RepUse:
 902             m_out.set(lowInt52(m_node->child1()), destination);
 903             break;
 904         case BooleanUse:
 905             m_out.set(lowBoolean(m_node->child1()), destination);
 906             break;
 907         case CellUse:
 908             m_out.set(lowCell(m_node->child1()), destination);
 909             break;
 910         case UntypedUse:
 911             m_out.set(lowJSValue(m_node->child1()), destination);
 912             break;
 913         default:
 914             DFG_CRASH(m_graph, m_node, "Bad use kind");
 915             break;
 916         }
 917     }
 918
 919     void compilePhi()
 920     {
 921         LValue source = m_phis.get(m_node);
 922
 923         switch (m_node->flags() & NodeResultMask) {
 924         case NodeResultDouble:
 925             setDouble(m_out.get(source));
 926             break;
 927         case NodeResultInt32:
 928             setInt32(m_out.get(source));
 929             break;
 930         case NodeResultInt52:
 931             setInt52(m_out.get(source));
 932             break;
 933         case NodeResultBoolean:
 934             setBoolean(m_out.get(source));
 935             break;
 936         case NodeResultJS:
 937             setJSValue(m_out.get(source));
 938             break;
 939         default:
 940             DFG_CRASH(m_graph, m_node, "Bad use kind");
 941             break;
 942         }
 943     }
 944
 945     void compileDoubleConstant()
 946     {
 947         setDouble(m_out.constDouble(m_node->asNumber()));
 948     }
 949
 950     void compileInt52Constant()
 951     {
 952         int64_t value = m_node->asMachineInt();
 953
 954         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
 955         setStrictInt52(m_out.constInt64(value));
 956     }
 957
 958     void compileDoubleRep()
 959     {
 960         switch (m_node->child1().useKind()) {
 961         case NumberUse: {
 962             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 963             setDouble(jsValueToDouble(m_node->child1(), value));
 964             return;
 965         }
 966
 967         case Int52RepUse: {
 968             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
 969             return;
 970         }
 971
 972         default:
 973             DFG_CRASH(m_graph, m_node, "Bad use kind");
 974         }
 975     }
 976
 977     void compileDoubleAsInt32()
 978     {
 979         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
 980         setInt32(integerValue);
 981     }
 982
 983     void compileValueRep()
 984     {
 985         switch (m_node->child1().useKind()) {
 986         case DoubleRepUse: {
 987             LValue value = lowDouble(m_node->child1());
 988
 989             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
 990                 value = m_out.select(
 991                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
 992             }
 993
 994             setJSValue(boxDouble(value));
 995             return;
 996         }
 997
 998         case Int52RepUse: {
 999             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1000             return;
1001         }
1002
1003         default:
1004             DFG_CRASH(m_graph, m_node, "Bad use kind");
1005         }
1006     }
1007
1008     void compileInt52Rep()
1009     {
1010         switch (m_node->child1().useKind()) {
1011         case Int32Use:
1012             setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
1013             return;
1014
1015         case MachineIntUse:
1016             setStrictInt52(
1017                 jsValueToStrictInt52(
1018                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1019             return;
1020
1021         case DoubleRepMachineIntUse:
1022             setStrictInt52(
1023                 doubleToStrictInt52(
1024                     m_node->child1(), lowDouble(m_node->child1())));
1025             return;
1026
1027         default:
1028             RELEASE_ASSERT_NOT_REACHED();
1029         }
1030     }
1031
1032     void compileValueToInt32()
1033     {
1034         switch (m_node->child1().useKind()) {
1035         case Int52RepUse:
1036             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1037             break;
1038
1039         case DoubleRepUse:
1040             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1041             break;
1042
1043         case NumberUse:
1044         case NotCellUse: {
1045             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1046             if (isValid(value)) {
1047                 setInt32(value.value());
1048                 break;
1049             }
1050
1051             value = m_jsValueValues.get(m_node->child1().node());
1052             if (isValid(value)) {
1053                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1054                 break;
1055             }
1056
1057             // We'll basically just get here for constants. But it's good to have this
1058             // catch-all since we often add new representations into the mix.
1059             setInt32(
1060                 numberOrNotCellToInt32(
1061                     m_node->child1(),
1062                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1063             break;
1064         }
1065
1066         default:
1067             DFG_CRASH(m_graph, m_node, "Bad use kind");
1068             break;
1069         }
1070     }
1071
1072     void compileBooleanToNumber()
1073     {
1074         switch (m_node->child1().useKind()) {
1075         case BooleanUse: {
1076             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1077             return;
1078         }
1079
1080         case UntypedUse: {
1081             LValue value = lowJSValue(m_node->child1());
1082
1083             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1084                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1085                 return;
1086             }
1087
1088             LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
1089             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
1090
1091             ValueFromBlock notBooleanResult = m_out.anchor(value);
1092             m_out.branch(
1093                 isBoolean(value, provenType(m_node->child1())),
1094                 unsure(booleanCase), unsure(continuation));
1095
1096             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1097             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1098                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1099             m_out.jump(continuation);
1100
1101             m_out.appendTo(continuation, lastNext);
1102             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1103             return;
1104         }
1105
1106         default:
1107             RELEASE_ASSERT_NOT_REACHED();
1108             return;
1109         }
1110     }
1111
1112     void compileExtractOSREntryLocal()
1113     {
1114         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1115             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1116         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1117     }
1118
1119     void compileGetStack()
1120     {
1121         // GetLocals arise only for captured variables and arguments. For arguments, we might have
1122         // already loaded it.
1123         if (LValue value = m_loadedArgumentValues.get(m_node)) {
1124             setJSValue(value);
1125             return;
1126         }
1127
1128         StackAccessData* data = m_node->stackAccessData();
1129         AbstractValue& value = m_state.variables().operand(data->local);
1130
1131         DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1132         DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1133
1134         if (isInt32Speculation(value.m_type))
1135             setInt32(m_out.load32(payloadFor(data->machineLocal)));
1136         else
1137             setJSValue(m_out.load64(addressFor(data->machineLocal)));
1138     }
1139
1140     void compilePutStack()
1141     {
1142         StackAccessData* data = m_node->stackAccessData();
1143         switch (data->format) {
1144         case FlushedJSValue: {
1145             LValue value = lowJSValue(m_node->child1());
1146             m_out.store64(value, addressFor(data->machineLocal));
1147             break;
1148         }
1149
1150         case FlushedDouble: {
1151             LValue value = lowDouble(m_node->child1());
1152             m_out.storeDouble(value, addressFor(data->machineLocal));
1153             break;
1154         }
1155
1156         case FlushedInt32: {
1157             LValue value = lowInt32(m_node->child1());
1158             m_out.store32(value, payloadFor(data->machineLocal));
1159             break;
1160         }
1161
1162         case FlushedInt52: {
1163             LValue value = lowInt52(m_node->child1());
1164             m_out.store64(value, addressFor(data->machineLocal));
1165             break;
1166         }
1167
1168         case FlushedCell: {
1169             LValue value = lowCell(m_node->child1());
1170             m_out.store64(value, addressFor(data->machineLocal));
1171             break;
1172         }
1173
1174         case FlushedBoolean: {
1175             speculateBoolean(m_node->child1());
1176             m_out.store64(
1177                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1178                 addressFor(data->machineLocal));
1179             break;
1180         }
1181
1182         default:
1183             DFG_CRASH(m_graph, m_node, "Bad flush format");
1184             break;
1185         }
1186     }
1187
1188     void compileNoOp()
1189     {
1190         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1191     }
1192
1193     void compileToThis()
1194     {
1195         LValue value = lowJSValue(m_node->child1());
1196
1197         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
1198         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
1199         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
1200
1201         m_out.branch(
1202             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1203
1204         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1205         ValueFromBlock fastResult = m_out.anchor(value);
1206         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1207
1208         m_out.appendTo(slowCase, continuation);
1209         J_JITOperation_EJ function;
1210         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1211             function = operationToThisStrict;
1212         else
1213             function = operationToThis;
1214         ValueFromBlock slowResult = m_out.anchor(
1215             vmCall(m_out.operation(function), m_callFrame, value));
1216         m_out.jump(continuation);
1217
1218         m_out.appendTo(continuation, lastNext);
1219         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1220     }
1221
1222     void compileValueAdd()
1223     {
1224         J_JITOperation_EJJ operation;
1225         if (!(provenType(m_node->child1()) & SpecFullNumber)
1226             && !(provenType(m_node->child2()) & SpecFullNumber))
1227             operation = operationValueAddNotNumber;
1228         else
1229             operation = operationValueAdd;
1230         setJSValue(vmCall(
1231             m_out.operation(operation), m_callFrame,
1232             lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1233     }
1234
1235     void compileArithAddOrSub()
1236     {
1237         bool isSub =  m_node->op() == ArithSub;
1238         switch (m_node->binaryUseKind()) {
1239         case Int32Use: {
1240             LValue left = lowInt32(m_node->child1());
1241             LValue right = lowInt32(m_node->child2());
1242
1243             if (!shouldCheckOverflow(m_node->arithMode())) {
1244                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1245                 break;
1246             }
1247
1248             LValue result;
1249             if (!isSub) {
1250                 result = m_out.addWithOverflow32(left, right);
1251
1252                 if (doesKill(m_node->child2())) {
1253                     addAvailableRecovery(
1254                         m_node->child2(), SubRecovery,
1255                         m_out.extractValue(result, 0), left, ValueFormatInt32);
1256                 } else if (doesKill(m_node->child1())) {
1257                     addAvailableRecovery(
1258                         m_node->child1(), SubRecovery,
1259                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1260                 }
1261             } else {
1262                 result = m_out.subWithOverflow32(left, right);
1263
1264                 if (doesKill(m_node->child2())) {
1265                     // result = left - right
1266                     // result - left = -right
1267                     // right = left - result
1268                     addAvailableRecovery(
1269                         m_node->child2(), SubRecovery,
1270                         left, m_out.extractValue(result, 0), ValueFormatInt32);
1271                 } else if (doesKill(m_node->child1())) {
1272                     // result = left - right
1273                     // result + right = left
1274                     addAvailableRecovery(
1275                         m_node->child1(), AddRecovery,
1276                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1277                 }
1278             }
1279
1280             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1281             setInt32(m_out.extractValue(result, 0));
1282             break;
1283         }
1284
1285         case Int52RepUse: {
1286             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)
1287                 && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) {
1288                 Int52Kind kind;
1289                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1290                 LValue right = lowInt52(m_node->child2(), kind);
1291                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1292                 break;
1293             }
1294
1295             LValue left = lowInt52(m_node->child1());
1296             LValue right = lowInt52(m_node->child2());
1297
1298             LValue result;
1299             if (!isSub) {
1300                 result = m_out.addWithOverflow64(left, right);
1301
1302                 if (doesKill(m_node->child2())) {
1303                     addAvailableRecovery(
1304                         m_node->child2(), SubRecovery,
1305                         m_out.extractValue(result, 0), left, ValueFormatInt52);
1306                 } else if (doesKill(m_node->child1())) {
1307                     addAvailableRecovery(
1308                         m_node->child1(), SubRecovery,
1309                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1310                 }
1311             } else {
1312                 result = m_out.subWithOverflow64(left, right);
1313
1314                 if (doesKill(m_node->child2())) {
1315                     // result = left - right
1316                     // result - left = -right
1317                     // right = left - result
1318                     addAvailableRecovery(
1319                         m_node->child2(), SubRecovery,
1320                         left, m_out.extractValue(result, 0), ValueFormatInt52);
1321                 } else if (doesKill(m_node->child1())) {
1322                     // result = left - right
1323                     // result + right = left
1324                     addAvailableRecovery(
1325                         m_node->child1(), AddRecovery,
1326                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1327                 }
1328             }
1329
1330             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1331             setInt52(m_out.extractValue(result, 0));
1332             break;
1333         }
1334
1335         case DoubleRepUse: {
1336             LValue C1 = lowDouble(m_node->child1());
1337             LValue C2 = lowDouble(m_node->child2());
1338
1339             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1340             break;
1341         }
1342
1343         default:
1344             DFG_CRASH(m_graph, m_node, "Bad use kind");
1345             break;
1346         }
1347     }
1348
1349     void compileArithClz32()
1350     {
1351         LValue operand = lowInt32(m_node->child1());
1352         LValue isZeroUndef = m_out.booleanFalse;
1353         setInt32(m_out.ctlz32(operand, isZeroUndef));
1354     }
1355
1356     void compileArithMul()
1357     {
1358         switch (m_node->binaryUseKind()) {
1359         case Int32Use: {
1360             LValue left = lowInt32(m_node->child1());
1361             LValue right = lowInt32(m_node->child2());
1362
1363             LValue result;
1364
1365             if (!shouldCheckOverflow(m_node->arithMode()))
1366                 result = m_out.mul(left, right);
1367             else {
1368                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1369                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1370                 result = m_out.extractValue(overflowResult, 0);
1371             }
1372
1373             if (shouldCheckNegativeZero(m_node->arithMode())) {
1374                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1375                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1376
1377                 m_out.branch(
1378                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1379
1380                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1381                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1382                 speculate(NegativeZero, noValue(), 0, cond);
1383                 m_out.jump(continuation);
1384                 m_out.appendTo(continuation, lastNext);
1385             }
1386
1387             setInt32(result);
1388             break;
1389         }
1390
1391         case Int52RepUse: {
1392             Int52Kind kind;
1393             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1394             LValue right = lowInt52(m_node->child2(), opposite(kind));
1395
1396             LValue overflowResult = m_out.mulWithOverflow64(left, right);
1397             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1398             LValue result = m_out.extractValue(overflowResult, 0);
1399
1400             if (shouldCheckNegativeZero(m_node->arithMode())) {
1401                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1402                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1403
1404                 m_out.branch(
1405                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1406
1407                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1408                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1409                 speculate(NegativeZero, noValue(), 0, cond);
1410                 m_out.jump(continuation);
1411                 m_out.appendTo(continuation, lastNext);
1412             }
1413
1414             setInt52(result);
1415             break;
1416         }
1417
1418         case DoubleRepUse: {
1419             setDouble(
1420                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1421             break;
1422         }
1423
1424         default:
1425             DFG_CRASH(m_graph, m_node, "Bad use kind");
1426             break;
1427         }
1428     }
1429
1430     void compileArithDiv()
1431     {
1432         switch (m_node->binaryUseKind()) {
1433         case Int32Use: {
1434             LValue numerator = lowInt32(m_node->child1());
1435             LValue denominator = lowInt32(m_node->child2());
1436
1437             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1438             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1439             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1440
1441             Vector<ValueFromBlock, 3> results;
1442
1443             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1444
1445             m_out.branch(
1446                 m_out.above(adjustedDenominator, m_out.int32One),
1447                 usually(continuation), rarely(unsafeDenominator));
1448
1449             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1450
1451             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1452
1453             if (shouldCheckOverflow(m_node->arithMode())) {
1454                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1455                 speculate(Overflow, noValue(), 0, cond);
1456                 m_out.jump(continuation);
1457             } else {
1458                 // This is the case where we convert the result to an int after we're done. So,
1459                 // if the denominator is zero, then the result should be zero.
1460                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1461                 // check above) and the numerator is -2^31 then the result should be -2^31.
1462
1463                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1464                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1465                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1466
1467                 m_out.branch(
1468                     m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1469
1470                 m_out.appendTo(divByZero, notDivByZero);
1471                 results.append(m_out.anchor(m_out.int32Zero));
1472                 m_out.jump(done);
1473
1474                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1475                 m_out.branch(
1476                     m_out.equal(numerator, neg2ToThe31),
1477                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1478
1479                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1480                 results.append(m_out.anchor(neg2ToThe31));
1481                 m_out.jump(done);
1482             }
1483
1484             m_out.appendTo(continuation, done);
1485
1486             if (shouldCheckNegativeZero(m_node->arithMode())) {
1487                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1488                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1489
1490                 m_out.branch(
1491                     m_out.isZero32(numerator),
1492                     rarely(zeroNumerator), usually(numeratorContinuation));
1493
1494                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1495
1496                 speculate(
1497                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1498
1499                 m_out.jump(numeratorContinuation);
1500
1501                 m_out.appendTo(numeratorContinuation, innerLastNext);
1502             }
1503
1504             LValue result = m_out.div(numerator, denominator);
1505
1506             if (shouldCheckOverflow(m_node->arithMode())) {
1507                 speculate(
1508                     Overflow, noValue(), 0,
1509                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1510             }
1511
1512             results.append(m_out.anchor(result));
1513             m_out.jump(done);
1514
1515             m_out.appendTo(done, lastNext);
1516
1517             setInt32(m_out.phi(m_out.int32, results));
1518             break;
1519         }
1520
1521         case DoubleRepUse: {
1522             setDouble(m_out.doubleDiv(
1523                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1524             break;
1525         }
1526
1527         default:
1528             DFG_CRASH(m_graph, m_node, "Bad use kind");
1529             break;
1530         }
1531     }
1532
1533     void compileArithMod()
1534     {
1535         switch (m_node->binaryUseKind()) {
1536         case Int32Use: {
1537             LValue numerator = lowInt32(m_node->child1());
1538             LValue denominator = lowInt32(m_node->child2());
1539
1540             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1541             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1542             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1543
1544             Vector<ValueFromBlock, 3> results;
1545
1546             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1547
1548             m_out.branch(
1549                 m_out.above(adjustedDenominator, m_out.int32One),
1550                 usually(continuation), rarely(unsafeDenominator));
1551
1552             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1553
1554             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1555
1556             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1557             // separate case for that. But it probably doesn't matter so much.
1558             if (shouldCheckOverflow(m_node->arithMode())) {
1559                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1560                 speculate(Overflow, noValue(), 0, cond);
1561                 m_out.jump(continuation);
1562             } else {
1563                 // This is the case where we convert the result to an int after we're done. So,
1564                 // if the denominator is zero, then the result should be result should be zero.
1565                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1566                 // check above) and the numerator is -2^31 then the result should be -2^31.
1567
1568                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1569                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1570                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1571
1572                 m_out.branch(
1573                     m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1574
1575                 m_out.appendTo(modByZero, notModByZero);
1576                 results.append(m_out.anchor(m_out.int32Zero));
1577                 m_out.jump(done);
1578
1579                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1580                 m_out.branch(
1581                     m_out.equal(numerator, neg2ToThe31),
1582                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1583
1584                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1585                 results.append(m_out.anchor(m_out.int32Zero));
1586                 m_out.jump(done);
1587             }
1588
1589             m_out.appendTo(continuation, done);
1590
1591             LValue remainder = m_out.rem(numerator, denominator);
1592
1593             if (shouldCheckNegativeZero(m_node->arithMode())) {
1594                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1595                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1596
1597                 m_out.branch(
1598                     m_out.lessThan(numerator, m_out.int32Zero),
1599                     unsure(negativeNumerator), unsure(numeratorContinuation));
1600
1601                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1602
1603                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1604
1605                 m_out.jump(numeratorContinuation);
1606
1607                 m_out.appendTo(numeratorContinuation, innerLastNext);
1608             }
1609
1610             results.append(m_out.anchor(remainder));
1611             m_out.jump(done);
1612
1613             m_out.appendTo(done, lastNext);
1614
1615             setInt32(m_out.phi(m_out.int32, results));
1616             break;
1617         }
1618
1619         case DoubleRepUse: {
1620             setDouble(
1621                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1622             break;
1623         }
1624
1625         default:
1626             DFG_CRASH(m_graph, m_node, "Bad use kind");
1627             break;
1628         }
1629     }
1630
1631     void compileArithMinOrMax()
1632     {
1633         switch (m_node->binaryUseKind()) {
1634         case Int32Use: {
1635             LValue left = lowInt32(m_node->child1());
1636             LValue right = lowInt32(m_node->child2());
1637
1638             setInt32(
1639                 m_out.select(
1640                     m_node->op() == ArithMin
1641                         ? m_out.lessThan(left, right)
1642                         : m_out.lessThan(right, left),
1643                     left, right));
1644             break;
1645         }
1646
1647         case DoubleRepUse: {
1648             LValue left = lowDouble(m_node->child1());
1649             LValue right = lowDouble(m_node->child2());
1650
1651             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1652             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1653
1654             Vector<ValueFromBlock, 2> results;
1655
1656             results.append(m_out.anchor(left));
1657             m_out.branch(
1658                 m_node->op() == ArithMin
1659                     ? m_out.doubleLessThan(left, right)
1660                     : m_out.doubleGreaterThan(left, right),
1661                 unsure(continuation), unsure(notLessThan));
1662
1663             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1664             results.append(m_out.anchor(m_out.select(
1665                 m_node->op() == ArithMin
1666                     ? m_out.doubleGreaterThanOrEqual(left, right)
1667                     : m_out.doubleLessThanOrEqual(left, right),
1668                 right, m_out.constDouble(PNaN))));
1669             m_out.jump(continuation);
1670
1671             m_out.appendTo(continuation, lastNext);
1672             setDouble(m_out.phi(m_out.doubleType, results));
1673             break;
1674         }
1675
1676         default:
1677             DFG_CRASH(m_graph, m_node, "Bad use kind");
1678             break;
1679         }
1680     }
1681
1682     void compileArithAbs()
1683     {
1684         switch (m_node->child1().useKind()) {
1685         case Int32Use: {
1686             LValue value = lowInt32(m_node->child1());
1687
1688             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1689             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1690
1691             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1692
1693             setInt32(result);
1694             break;
1695         }
1696
1697         case DoubleRepUse: {
1698             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1699             break;
1700         }
1701
1702         default:
1703             DFG_CRASH(m_graph, m_node, "Bad use kind");
1704             break;
1705         }
1706     }
1707
1708     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1709
1710     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1711
1712     void compileArithPow()
1713     {
1714         // FIXME: investigate llvm.powi to better understand its performance characteristics.
1715         // It might be better to have the inline loop in DFG too.
1716         if (m_node->child2().useKind() == Int32Use)
1717             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1718         else {
1719             LValue base = lowDouble(m_node->child1());
1720             LValue exponent = lowDouble(m_node->child2());
1721
1722             LBasicBlock integerExponentIsSmallBlock = FTL_NEW_BLOCK(m_out, ("ArithPow test integer exponent is small."));
1723             LBasicBlock integerExponentPowBlock = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, (int)double)."));
1724             LBasicBlock doubleExponentPowBlockEntry = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, double)."));
1725             LBasicBlock nanExceptionExponentIsInfinity = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check exponent is infinity."));
1726             LBasicBlock nanExceptionBaseIsOne = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check base is one."));
1727             LBasicBlock powBlock = FTL_NEW_BLOCK(m_out, ("ArithPow regular pow"));
1728             LBasicBlock nanExceptionResultIsNaN = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, result is NaN."));
1729             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithPow continuation"));
1730
1731             LValue integerExponent = m_out.fpToInt32(exponent);
1732             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1733             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1734             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1735
1736             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1737             LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1738             m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1739
1740             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1741             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1742             m_out.jump(continuation);
1743
1744             // If y is NaN, the result is NaN.
1745             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1746             LValue exponentIsNaN;
1747             if (provenType(m_node->child2()) & SpecDoubleNaN)
1748                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1749             else
1750                 exponentIsNaN = m_out.booleanFalse;
1751             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1752
1753             // If abs(x) is 1 and y is +infinity, the result is NaN.
1754             // If abs(x) is 1 and y is -infinity, the result is NaN.
1755             m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1756             LValue absoluteExponent = m_out.doubleAbs(exponent);
1757             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1758             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1759
1760             m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1761             LValue absoluteBase = m_out.doubleAbs(base);
1762             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1763             m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1764
1765             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1766             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1767             m_out.jump(continuation);
1768
1769             m_out.appendTo(nanExceptionResultIsNaN, continuation);
1770             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1771             m_out.jump(continuation);
1772
1773             m_out.appendTo(continuation, lastNext);
1774             setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1775         }
1776     }
1777
1778     void compileArithRound()
1779     {
1780         LBasicBlock realPartIsMoreThanHalf = FTL_NEW_BLOCK(m_out, ("ArithRound should round down"));
1781         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithRound continuation"));
1782
1783         LValue value = lowDouble(m_node->child1());
1784         LValue integerValue = m_out.ceil64(value);
1785         ValueFromBlock integerValueResult = m_out.anchor(integerValue);
1786
1787         LValue realPart = m_out.doubleSub(integerValue, value);
1788
1789         m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
1790
1791         LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
1792         LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
1793         ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
1794         m_out.jump(continuation);
1795         m_out.appendTo(continuation, lastNext);
1796
1797         LValue result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult);
1798
1799         if (producesInteger(m_node->arithRoundingMode())) {
1800             LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
1801             setInt32(integerValue);
1802         } else
1803             setDouble(result);
1804     }
1805
1806     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1807
1808     void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
1809
1810     void compileArithFRound()
1811     {
1812         LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1813         setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1814     }
1815
1816     void compileArithNegate()
1817     {
1818         switch (m_node->child1().useKind()) {
1819         case Int32Use: {
1820             LValue value = lowInt32(m_node->child1());
1821
1822             LValue result;
1823             if (!shouldCheckOverflow(m_node->arithMode()))
1824                 result = m_out.neg(value);
1825             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1826                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1827                 // does the trick, though.
1828                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1829                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1830                 result = m_out.extractValue(overflowResult, 0);
1831             } else {
1832                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1833                 result = m_out.neg(value);
1834             }
1835
1836             setInt32(result);
1837             break;
1838         }
1839
1840         case Int52RepUse: {
1841             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) {
1842                 Int52Kind kind;
1843                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1844                 LValue result = m_out.neg(value);
1845                 if (shouldCheckNegativeZero(m_node->arithMode()))
1846                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1847                 setInt52(result, kind);
1848                 break;
1849             }
1850
1851             LValue value = lowInt52(m_node->child1());
1852             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1853             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1854             LValue result = m_out.extractValue(overflowResult, 0);
1855             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1856             setInt52(result);
1857             break;
1858         }
1859
1860         case DoubleRepUse: {
1861             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1862             break;
1863         }
1864
1865         default:
1866             DFG_CRASH(m_graph, m_node, "Bad use kind");
1867             break;
1868         }
1869     }
1870
1871     void compileBitAnd()
1872     {
1873         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1874     }
1875
1876     void compileBitOr()
1877     {
1878         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1879     }
1880
1881     void compileBitXor()
1882     {
1883         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1884     }
1885
1886     void compileBitRShift()
1887     {
1888         setInt32(m_out.aShr(
1889             lowInt32(m_node->child1()),
1890             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1891     }
1892
1893     void compileBitLShift()
1894     {
1895         setInt32(m_out.shl(
1896             lowInt32(m_node->child1()),
1897             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1898     }
1899
1900     void compileBitURShift()
1901     {
1902         setInt32(m_out.lShr(
1903             lowInt32(m_node->child1()),
1904             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1905     }
1906
1907     void compileUInt32ToNumber()
1908     {
1909         LValue value = lowInt32(m_node->child1());
1910
1911         if (doesOverflow(m_node->arithMode())) {
1912             setDouble(m_out.unsignedToDouble(value));
1913             return;
1914         }
1915
1916         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
1917         setInt32(value);
1918     }
1919
1920     void compileCheckStructure()
1921     {
1922         LValue cell = lowCell(m_node->child1());
1923
1924         ExitKind exitKind;
1925         if (m_node->child1()->hasConstant())
1926             exitKind = BadConstantCache;
1927         else
1928             exitKind = BadCache;
1929
1930         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1931
1932         checkStructure(
1933             structureID, jsValueValue(cell), exitKind, m_node->structureSet(),
1934             [this] (Structure* structure) {
1935                 return weakStructureID(structure);
1936             });
1937     }
1938
1939     void compileCheckCell()
1940     {
1941         LValue cell = lowCell(m_node->child1());
1942
1943         speculate(
1944             BadCell, jsValueValue(cell), m_node->child1().node(),
1945             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
1946     }
1947
1948     void compileCheckBadCell()
1949     {
1950         terminate(BadCell);
1951     }
1952
1953     void compileCheckNotEmpty()
1954     {
1955         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
1956     }
1957
1958     void compileGetExecutable()
1959     {
1960         LValue cell = lowCell(m_node->child1());
1961         speculateFunction(m_node->child1(), cell);
1962         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
1963     }
1964
1965     void compileArrayifyToStructure()
1966     {
1967         LValue cell = lowCell(m_node->child1());
1968         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1969
1970         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1971         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1972
1973         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1974
1975         m_out.branch(
1976             m_out.notEqual(structureID, weakStructureID(m_node->structure())),
1977             rarely(unexpectedStructure), usually(continuation));
1978
1979         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1980
1981         if (property) {
1982             switch (m_node->arrayMode().type()) {
1983             case Array::Int32:
1984             case Array::Double:
1985             case Array::Contiguous:
1986                 speculate(
1987                     Uncountable, noValue(), 0,
1988                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1989                 break;
1990             default:
1991                 break;
1992             }
1993         }
1994
1995         switch (m_node->arrayMode().type()) {
1996         case Array::Int32:
1997             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1998             break;
1999         case Array::Double:
2000             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
2001             break;
2002         case Array::Contiguous:
2003             vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
2004             break;
2005         case Array::ArrayStorage:
2006         case Array::SlowPutArrayStorage:
2007             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
2008             break;
2009         default:
2010             DFG_CRASH(m_graph, m_node, "Bad array type");
2011             break;
2012         }
2013
2014         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2015         speculate(
2016             BadIndexingType, jsValueValue(cell), 0,
2017             m_out.notEqual(structureID, weakStructureID(m_node->structure())));
2018         m_out.jump(continuation);
2019
2020         m_out.appendTo(continuation, lastNext);
2021     }
2022
2023     void compilePutStructure()
2024     {
2025         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
2026
2027         Structure* oldStructure = m_node->transition()->previous;
2028         Structure* newStructure = m_node->transition()->next;
2029         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
2030         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
2031         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
2032
2033         LValue cell = lowCell(m_node->child1());
2034         m_out.store32(
2035             weakStructureID(newStructure),
2036             cell, m_heaps.JSCell_structureID);
2037     }
2038
2039     void compileGetById()
2040     {
2041         // Pretty much the only reason why we don't also support GetByIdFlush is because:
2042         // https://bugs.webkit.org/show_bug.cgi?id=125711
2043
2044         switch (m_node->child1().useKind()) {
2045         case CellUse: {
2046             setJSValue(getById(lowCell(m_node->child1())));
2047             return;
2048         }
2049
2050         case UntypedUse: {
2051             // This is pretty weird, since we duplicate the slow path both here and in the
2052             // code generated by the IC. We should investigate making this less bad.
2053             // https://bugs.webkit.org/show_bug.cgi?id=127830
2054             LValue value = lowJSValue(m_node->child1());
2055
2056             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
2057             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
2058             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
2059
2060             m_out.branch(
2061                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2062
2063             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2064             ValueFromBlock cellResult = m_out.anchor(getById(value));
2065             m_out.jump(continuation);
2066
2067             m_out.appendTo(notCellCase, continuation);
2068             ValueFromBlock notCellResult = m_out.anchor(vmCall(
2069                 m_out.operation(operationGetById),
2070                 m_callFrame, getUndef(m_out.intPtr), value,
2071                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2072             m_out.jump(continuation);
2073
2074             m_out.appendTo(continuation, lastNext);
2075             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2076             return;
2077         }
2078
2079         default:
2080             DFG_CRASH(m_graph, m_node, "Bad use kind");
2081             return;
2082         }
2083     }
2084
2085     void compilePutById()
2086     {
2087         // See above; CellUse is easier so we do only that for now.
2088         ASSERT(m_node->child1().useKind() == CellUse);
2089
2090         LValue base = lowCell(m_node->child1());
2091         LValue value = lowJSValue(m_node->child2());
2092         auto uid = m_graph.identifiers()[m_node->identifierNumber()];
2093
2094         // Arguments: id, bytes, target, numArgs, args...
2095         unsigned stackmapID = m_stackmapIDs++;
2096
2097         if (verboseCompilationEnabled())
2098             dataLog("    Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
2099
2100         LValue call = m_out.call(
2101             m_out.patchpointVoidIntrinsic(),
2102             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
2103             constNull(m_out.ref8), m_out.constInt32(2), base, value);
2104         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
2105
2106         m_ftlState.putByIds.append(PutByIdDescriptor(
2107             stackmapID, m_node->origin.semantic, uid,
2108             m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
2109             m_node->op() == PutByIdDirect ? Direct : NotDirect));
2110     }
2111
2112     void compileGetButterfly()
2113     {
2114         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2115     }
2116
2117     void compileConstantStoragePointer()
2118     {
2119         setStorage(m_out.constIntPtr(m_node->storagePointer()));
2120     }
2121
2122     void compileGetIndexedPropertyStorage()
2123     {
2124         LValue cell = lowCell(m_node->child1());
2125
2126         if (m_node->arrayMode().type() == Array::String) {
2127             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
2128             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
2129
2130             ValueFromBlock fastResult = m_out.anchor(
2131                 m_out.loadPtr(cell, m_heaps.JSString_value));
2132
2133             m_out.branch(
2134                 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
2135
2136             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2137
2138             ValueFromBlock slowResult = m_out.anchor(
2139                 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
2140
2141             m_out.jump(continuation);
2142
2143             m_out.appendTo(continuation, lastNext);
2144
2145             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2146             return;
2147         }
2148
2149         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2150     }
2151
2152     void compileCheckArray()
2153     {
2154         Edge edge = m_node->child1();
2155         LValue cell = lowCell(edge);
2156
2157         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
2158             return;
2159
2160         speculate(
2161             BadIndexingType, jsValueValue(cell), 0,
2162             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
2163     }
2164
2165     void compileGetTypedArrayByteOffset()
2166     {
2167         LValue basePtr = lowCell(m_node->child1());
2168
2169         LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
2170         LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
2171         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
2172
2173         LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2174         m_out.branch(
2175             m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2176             unsure(simpleCase), unsure(wastefulCase));
2177
2178         // begin simple case
2179         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2180
2181         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2182
2183         m_out.jump(continuation);
2184
2185         // begin wasteful case
2186         m_out.appendTo(wastefulCase, continuation);
2187
2188         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2189         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2190         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2191         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2192
2193         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2194
2195         m_out.jump(continuation);
2196         m_out.appendTo(continuation, lastNext);
2197
2198         // output
2199         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2200     }
2201
2202     void compileGetArrayLength()
2203     {
2204         switch (m_node->arrayMode().type()) {
2205         case Array::Int32:
2206         case Array::Double:
2207         case Array::Contiguous: {
2208             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2209             return;
2210         }
2211
2212         case Array::String: {
2213             LValue string = lowCell(m_node->child1());
2214             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2215             return;
2216         }
2217
2218         case Array::DirectArguments: {
2219             LValue arguments = lowCell(m_node->child1());
2220             speculate(
2221                 ExoticObjectMode, noValue(), nullptr,
2222                 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2223             setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2224             return;
2225         }
2226
2227         case Array::ScopedArguments: {
2228             LValue arguments = lowCell(m_node->child1());
2229             speculate(
2230                 ExoticObjectMode, noValue(), nullptr,
2231                 m_out.notZero8(m_out.load8(arguments, m_heaps.ScopedArguments_overrodeThings)));
2232             setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2233             return;
2234         }
2235
2236         default:
2237             if (isTypedView(m_node->arrayMode().typedArrayType())) {
2238                 setInt32(
2239                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2240                 return;
2241             }
2242
2243             DFG_CRASH(m_graph, m_node, "Bad array type");
2244             return;
2245         }
2246     }
2247
2248     void compileCheckInBounds()
2249     {
2250         speculate(
2251             OutOfBounds, noValue(), 0,
2252             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2253     }
2254
2255     void compileGetByVal()
2256     {
2257         switch (m_node->arrayMode().type()) {
2258         case Array::Int32:
2259         case Array::Contiguous: {
2260             LValue index = lowInt32(m_node->child2());
2261             LValue storage = lowStorage(m_node->child3());
2262
2263             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2264                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2265
2266             if (m_node->arrayMode().isInBounds()) {
2267                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2268                 LValue isHole = m_out.isZero64(result);
2269                 if (m_node->arrayMode().isSaneChain()) {
2270                     DFG_ASSERT(
2271                         m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous);
2272                     result = m_out.select(
2273                         isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
2274                 } else
2275                     speculate(LoadFromHole, noValue(), 0, isHole);
2276                 setJSValue(result);
2277                 return;
2278             }
2279
2280             LValue base = lowCell(m_node->child1());
2281
2282             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2283             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2284             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2285
2286             m_out.branch(
2287                 m_out.aboveOrEqual(
2288                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2289                 rarely(slowCase), usually(fastCase));
2290
2291             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2292
2293             ValueFromBlock fastResult = m_out.anchor(
2294                 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2295             m_out.branch(
2296                 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2297
2298             m_out.appendTo(slowCase, continuation);
2299             ValueFromBlock slowResult = m_out.anchor(
2300                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2301             m_out.jump(continuation);
2302
2303             m_out.appendTo(continuation, lastNext);
2304             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2305             return;
2306         }
2307
2308         case Array::Double: {
2309             LValue index = lowInt32(m_node->child2());
2310             LValue storage = lowStorage(m_node->child3());
2311
2312             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2313
2314             if (m_node->arrayMode().isInBounds()) {
2315                 LValue result = m_out.loadDouble(
2316                     baseIndex(heap, storage, index, m_node->child2()));
2317
2318                 if (!m_node->arrayMode().isSaneChain()) {
2319                     speculate(
2320                         LoadFromHole, noValue(), 0,
2321                         m_out.doubleNotEqualOrUnordered(result, result));
2322                 }
2323                 setDouble(result);
2324                 break;
2325             }
2326
2327             LValue base = lowCell(m_node->child1());
2328
2329             LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2330             LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2331             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2332             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2333
2334             m_out.branch(
2335                 m_out.aboveOrEqual(
2336                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2337                 rarely(slowCase), usually(inBounds));
2338
2339             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2340             LValue doubleValue = m_out.loadDouble(
2341                 baseIndex(heap, storage, index, m_node->child2()));
2342             m_out.branch(
2343                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2344                 rarely(slowCase), usually(boxPath));
2345
2346             m_out.appendTo(boxPath, slowCase);
2347             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2348             m_out.jump(continuation);
2349
2350             m_out.appendTo(slowCase, continuation);
2351             ValueFromBlock slowResult = m_out.anchor(
2352                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2353             m_out.jump(continuation);
2354
2355             m_out.appendTo(continuation, lastNext);
2356             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2357             return;
2358         }
2359
2360         case Array::DirectArguments: {
2361             LValue base = lowCell(m_node->child1());
2362             LValue index = lowInt32(m_node->child2());
2363
2364             speculate(
2365                 ExoticObjectMode, noValue(), nullptr,
2366                 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2367             speculate(
2368                 ExoticObjectMode, noValue(), nullptr,
2369                 m_out.aboveOrEqual(
2370                     index,
2371                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2372
2373             TypedPointer address = m_out.baseIndex(
2374                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2375             setJSValue(m_out.load64(address));
2376             return;
2377         }
2378
2379         case Array::ScopedArguments: {
2380             LValue base = lowCell(m_node->child1());
2381             LValue index = lowInt32(m_node->child2());
2382
2383             speculate(
2384                 ExoticObjectMode, noValue(), nullptr,
2385                 m_out.aboveOrEqual(
2386                     index,
2387                     m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2388
2389             LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2390             LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2391
2392             LBasicBlock namedCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments named case"));
2393             LBasicBlock overflowCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments overflow case"));
2394             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments continuation"));
2395
2396             m_out.branch(
2397                 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2398
2399             LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2400
2401             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2402             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2403
2404             TypedPointer address = m_out.baseIndex(
2405                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2406             LValue scopeOffset = m_out.load32(address);
2407
2408             speculate(
2409                 ExoticObjectMode, noValue(), nullptr,
2410                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2411
2412             address = m_out.baseIndex(
2413                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2414             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2415             m_out.jump(continuation);
2416
2417             m_out.appendTo(overflowCase, continuation);
2418
2419             address = m_out.baseIndex(
2420                 m_heaps.ScopedArguments_overflowStorage, base,
2421                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2422             LValue overflowValue = m_out.load64(address);
2423             speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2424             ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2425             m_out.jump(continuation);
2426
2427             m_out.appendTo(continuation, lastNext);
2428             setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2429             return;
2430         }
2431
2432         case Array::Generic: {
2433             setJSValue(vmCall(
2434                 m_out.operation(operationGetByVal), m_callFrame,
2435                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2436             return;
2437         }
2438
2439         case Array::String: {
2440             compileStringCharAt();
2441             return;
2442         }
2443
2444         default: {
2445             LValue index = lowInt32(m_node->child2());
2446             LValue storage = lowStorage(m_node->child3());
2447
2448             TypedArrayType type = m_node->arrayMode().typedArrayType();
2449
2450             if (isTypedView(type)) {
2451                 TypedPointer pointer = TypedPointer(
2452                     m_heaps.typedArrayProperties,
2453                     m_out.add(
2454                         storage,
2455                         m_out.shl(
2456                             m_out.zeroExtPtr(index),
2457                             m_out.constIntPtr(logElementSize(type)))));
2458
2459                 if (isInt(type)) {
2460                     LValue result;
2461                     switch (elementSize(type)) {
2462                     case 1:
2463                         result = m_out.load8(pointer);
2464                         break;
2465                     case 2:
2466                         result = m_out.load16(pointer);
2467                         break;
2468                     case 4:
2469                         result = m_out.load32(pointer);
2470                         break;
2471                     default:
2472                         DFG_CRASH(m_graph, m_node, "Bad element size");
2473                     }
2474
2475                     if (elementSize(type) < 4) {
2476                         if (isSigned(type))
2477                             result = m_out.signExt(result, m_out.int32);
2478                         else
2479                             result = m_out.zeroExt(result, m_out.int32);
2480                         setInt32(result);
2481                         return;
2482                     }
2483
2484                     if (isSigned(type)) {
2485                         setInt32(result);
2486                         return;
2487                     }
2488
2489                     if (m_node->shouldSpeculateInt32()) {
2490                         speculate(
2491                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2492                         setInt32(result);
2493                         return;
2494                     }
2495
2496                     if (m_node->shouldSpeculateMachineInt()) {
2497                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2498                         return;
2499                     }
2500
2501                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2502                     return;
2503                 }
2504
2505                 ASSERT(isFloat(type));
2506
2507                 LValue result;
2508                 switch (type) {
2509                 case TypeFloat32:
2510                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2511                     break;
2512                 case TypeFloat64:
2513                     result = m_out.loadDouble(pointer);
2514                     break;
2515                 default:
2516                     DFG_CRASH(m_graph, m_node, "Bad typed array type");
2517                 }
2518
2519                 setDouble(result);
2520                 return;
2521             }
2522
2523             DFG_CRASH(m_graph, m_node, "Bad array type");
2524             return;
2525         } }
2526     }
2527
2528     void compileGetMyArgumentByVal()
2529     {
2530         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2531
2532         LValue index = lowInt32(m_node->child2());
2533
2534         LValue limit;
2535         if (inlineCallFrame && !inlineCallFrame->isVarargs())
2536             limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2537         else {
2538             VirtualRegister argumentCountRegister;
2539             if (!inlineCallFrame)
2540                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2541             else
2542                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2543             limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2544         }
2545
2546         speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2547
2548         TypedPointer base;
2549         if (inlineCallFrame) {
2550             if (inlineCallFrame->arguments.size() <= 1) {
2551                 // We should have already exited due to the bounds check, above. Just tell the
2552                 // compiler that anything dominated by this instruction is not reachable, so
2553                 // that we don't waste time generating such code. This will also plant some
2554                 // kind of crashing instruction so that if by some fluke the bounds check didn't
2555                 // work, we'll crash in an easy-to-see way.
2556                 didAlreadyTerminate();
2557                 return;
2558             }
2559             base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2560         } else
2561             base = addressFor(virtualRegisterForArgument(1));
2562
2563         LValue pointer = m_out.baseIndex(
2564             base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2565         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2566     }
2567
2568     void compilePutByVal()
2569     {
2570         Edge child1 = m_graph.varArgChild(m_node, 0);
2571         Edge child2 = m_graph.varArgChild(m_node, 1);
2572         Edge child3 = m_graph.varArgChild(m_node, 2);
2573         Edge child4 = m_graph.varArgChild(m_node, 3);
2574         Edge child5 = m_graph.varArgChild(m_node, 4);
2575
2576         switch (m_node->arrayMode().type()) {
2577         case Array::Generic: {
2578             V_JITOperation_EJJJ operation;
2579             if (m_node->op() == PutByValDirect) {
2580                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2581                     operation = operationPutByValDirectStrict;
2582                 else
2583                     operation = operationPutByValDirectNonStrict;
2584             } else {
2585                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2586                     operation = operationPutByValStrict;
2587                 else
2588                     operation = operationPutByValNonStrict;
2589             }
2590
2591             vmCall(
2592                 m_out.operation(operation), m_callFrame,
2593                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2594             return;
2595         }
2596
2597         default:
2598             break;
2599         }
2600
2601         LValue base = lowCell(child1);
2602         LValue index = lowInt32(child2);
2603         LValue storage = lowStorage(child4);
2604
2605         switch (m_node->arrayMode().type()) {
2606         case Array::Int32:
2607         case Array::Double:
2608         case Array::Contiguous: {
2609             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2610             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2611
2612             switch (m_node->arrayMode().type()) {
2613             case Array::Int32:
2614             case Array::Contiguous: {
2615                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2616
2617                 if (m_node->arrayMode().type() == Array::Int32)
2618                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2619
2620                 TypedPointer elementPointer = m_out.baseIndex(
2621                     m_node->arrayMode().type() == Array::Int32 ?
2622                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2623                     storage, m_out.zeroExtPtr(index), provenValue(child2));
2624
2625                 if (m_node->op() == PutByValAlias) {
2626                     m_out.store64(value, elementPointer);
2627                     break;
2628                 }
2629
2630                 contiguousPutByValOutOfBounds(
2631                     codeBlock()->isStrictMode()
2632                     ? operationPutByValBeyondArrayBoundsStrict
2633                     : operationPutByValBeyondArrayBoundsNonStrict,
2634                     base, storage, index, value, continuation);
2635
2636                 m_out.store64(value, elementPointer);
2637                 break;
2638             }
2639
2640             case Array::Double: {
2641                 LValue value = lowDouble(child3);
2642
2643                 FTL_TYPE_CHECK(
2644                     doubleValue(value), child3, SpecDoubleReal,
2645                     m_out.doubleNotEqualOrUnordered(value, value));
2646
2647                 TypedPointer elementPointer = m_out.baseIndex(
2648                     m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2649                     provenValue(child2));
2650
2651                 if (m_node->op() == PutByValAlias) {
2652                     m_out.storeDouble(value, elementPointer);
2653                     break;
2654                 }
2655
2656                 contiguousPutByValOutOfBounds(
2657                     codeBlock()->isStrictMode()
2658                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2659                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2660                     base, storage, index, value, continuation);
2661
2662                 m_out.storeDouble(value, elementPointer);
2663                 break;
2664             }
2665
2666             default:
2667                 DFG_CRASH(m_graph, m_node, "Bad array type");
2668             }
2669
2670             m_out.jump(continuation);
2671             m_out.appendTo(continuation, outerLastNext);
2672             return;
2673         }
2674
2675         default:
2676             TypedArrayType type = m_node->arrayMode().typedArrayType();
2677
2678             if (isTypedView(type)) {
2679                 TypedPointer pointer = TypedPointer(
2680                     m_heaps.typedArrayProperties,
2681                     m_out.add(
2682                         storage,
2683                         m_out.shl(
2684                             m_out.zeroExt(index, m_out.intPtr),
2685                             m_out.constIntPtr(logElementSize(type)))));
2686
2687                 LType refType;
2688                 LValue valueToStore;
2689
2690                 if (isInt(type)) {
2691                     LValue intValue;
2692                     switch (child3.useKind()) {
2693                     case Int52RepUse:
2694                     case Int32Use: {
2695                         if (child3.useKind() == Int32Use)
2696                             intValue = lowInt32(child3);
2697                         else
2698                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2699
2700                         if (isClamped(type)) {
2701                             ASSERT(elementSize(type) == 1);
2702
2703                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2704                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2705
2706                             Vector<ValueFromBlock, 2> intValues;
2707                             intValues.append(m_out.anchor(m_out.int32Zero));
2708                             m_out.branch(
2709                                 m_out.lessThan(intValue, m_out.int32Zero),
2710                                 unsure(continuation), unsure(atLeastZero));
2711
2712                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2713
2714                             intValues.append(m_out.anchor(m_out.select(
2715                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
2716                                 m_out.constInt32(255),
2717                                 intValue)));
2718                             m_out.jump(continuation);
2719
2720                             m_out.appendTo(continuation, lastNext);
2721                             intValue = m_out.phi(m_out.int32, intValues);
2722                         }
2723                         break;
2724                     }
2725
2726                     case DoubleRepUse: {
2727                         LValue doubleValue = lowDouble(child3);
2728
2729                         if (isClamped(type)) {
2730                             ASSERT(elementSize(type) == 1);
2731
2732                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2733                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2734                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2735
2736                             Vector<ValueFromBlock, 3> intValues;
2737                             intValues.append(m_out.anchor(m_out.int32Zero));
2738                             m_out.branch(
2739                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2740                                 unsure(continuation), unsure(atLeastZero));
2741
2742                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2743                             intValues.append(m_out.anchor(m_out.constInt32(255)));
2744                             m_out.branch(
2745                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2746                                 unsure(continuation), unsure(withinRange));
2747
2748                             m_out.appendTo(withinRange, continuation);
2749                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2750                             m_out.jump(continuation);
2751
2752                             m_out.appendTo(continuation, lastNext);
2753                             intValue = m_out.phi(m_out.int32, intValues);
2754                         } else
2755                             intValue = doubleToInt32(doubleValue);
2756                         break;
2757                     }
2758
2759                     default:
2760                         DFG_CRASH(m_graph, m_node, "Bad use kind");
2761                     }
2762
2763                     switch (elementSize(type)) {
2764                     case 1:
2765                         valueToStore = m_out.intCast(intValue, m_out.int8);
2766                         refType = m_out.ref8;
2767                         break;
2768                     case 2:
2769                         valueToStore = m_out.intCast(intValue, m_out.int16);
2770                         refType = m_out.ref16;
2771                         break;
2772                     case 4:
2773                         valueToStore = intValue;
2774                         refType = m_out.ref32;
2775                         break;
2776                     default:
2777                         DFG_CRASH(m_graph, m_node, "Bad element size");
2778                     }
2779                 } else /* !isInt(type) */ {
2780                     LValue value = lowDouble(child3);
2781                     switch (type) {
2782                     case TypeFloat32:
2783                         valueToStore = m_out.fpCast(value, m_out.floatType);
2784                         refType = m_out.refFloat;
2785                         break;
2786                     case TypeFloat64:
2787                         valueToStore = value;
2788                         refType = m_out.refDouble;
2789                         break;
2790                     default:
2791                         DFG_CRASH(m_graph, m_node, "Bad typed array type");
2792                     }
2793                 }
2794
2795                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2796                     m_out.store(valueToStore, pointer, refType);
2797                 else {
2798                     LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2799                     LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2800
2801                     m_out.branch(
2802                         m_out.aboveOrEqual(index, lowInt32(child5)),
2803                         unsure(continuation), unsure(isInBounds));
2804
2805                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2806                     m_out.store(valueToStore, pointer, refType);
2807                     m_out.jump(continuation);
2808
2809                     m_out.appendTo(continuation, lastNext);
2810                 }
2811
2812                 return;
2813             }
2814
2815             DFG_CRASH(m_graph, m_node, "Bad array type");
2816             break;
2817         }
2818     }
2819
2820     void compileArrayPush()
2821     {
2822         LValue base = lowCell(m_node->child1());
2823         LValue storage = lowStorage(m_node->child3());
2824
2825         switch (m_node->arrayMode().type()) {
2826         case Array::Int32:
2827         case Array::Contiguous:
2828         case Array::Double: {
2829             LValue value;
2830             LType refType;
2831
2832             if (m_node->arrayMode().type() != Array::Double) {
2833                 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2834                 if (m_node->arrayMode().type() == Array::Int32) {
2835                     FTL_TYPE_CHECK(
2836                         jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2837                 }
2838                 refType = m_out.ref64;
2839             } else {
2840                 value = lowDouble(m_node->child2());
2841                 FTL_TYPE_CHECK(
2842                     doubleValue(value), m_node->child2(), SpecDoubleReal,
2843                     m_out.doubleNotEqualOrUnordered(value, value));
2844                 refType = m_out.refDouble;
2845             }
2846
2847             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2848
2849             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2850
2851             LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2852             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2853             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2854
2855             m_out.branch(
2856                 m_out.aboveOrEqual(
2857                     prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2858                 rarely(slowPath), usually(fastPath));
2859
2860             LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2861             m_out.store(
2862                 value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), refType);
2863             LValue newLength = m_out.add(prevLength, m_out.int32One);
2864             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2865
2866             ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2867             m_out.jump(continuation);
2868
2869             m_out.appendTo(slowPath, continuation);
2870             LValue operation;
2871             if (m_node->arrayMode().type() != Array::Double)
2872                 operation = m_out.operation(operationArrayPush);
2873             else
2874                 operation = m_out.operation(operationArrayPushDouble);
2875             ValueFromBlock slowResult = m_out.anchor(
2876                 vmCall(operation, m_callFrame, value, base));
2877             m_out.jump(continuation);
2878
2879             m_out.appendTo(continuation, lastNext);
2880             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2881             return;
2882         }
2883
2884         default:
2885             DFG_CRASH(m_graph, m_node, "Bad array type");
2886             return;
2887         }
2888     }
2889
2890     void compileArrayPop()
2891     {
2892         LValue base = lowCell(m_node->child1());
2893         LValue storage = lowStorage(m_node->child2());
2894
2895         switch (m_node->arrayMode().type()) {
2896         case Array::Int32:
2897         case Array::Double:
2898         case Array::Contiguous: {
2899             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2900
2901             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
2902             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
2903             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
2904
2905             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2906
2907             Vector<ValueFromBlock, 3> results;
2908             results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
2909             m_out.branch(
2910                 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
2911
2912             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2913             LValue newLength = m_out.sub(prevLength, m_out.int32One);
2914             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2915             TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength));
2916             if (m_node->arrayMode().type() != Array::Double) {
2917                 LValue result = m_out.load64(pointer);
2918                 m_out.store64(m_out.int64Zero, pointer);
2919                 results.append(m_out.anchor(result));
2920                 m_out.branch(
2921                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2922             } else {
2923                 LValue result = m_out.loadDouble(pointer);
2924                 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
2925                 results.append(m_out.anchor(boxDouble(result)));
2926                 m_out.branch(
2927                     m_out.doubleEqual(result, result),
2928                     usually(continuation), rarely(slowCase));
2929             }
2930
2931             m_out.appendTo(slowCase, continuation);
2932             results.append(m_out.anchor(vmCall(
2933                 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
2934             m_out.jump(continuation);
2935
2936             m_out.appendTo(continuation, lastNext);
2937             setJSValue(m_out.phi(m_out.int64, results));
2938             return;
2939         }
2940
2941         default:
2942             DFG_CRASH(m_graph, m_node, "Bad array type");
2943             return;
2944         }
2945     }
2946
2947     void compileCreateActivation()
2948     {
2949         LValue scope = lowCell(m_node->child1());
2950         SymbolTable* table = m_node->castOperand<SymbolTable*>();
2951         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
2952
2953         if (table->singletonScope()->isStillValid()) {
2954             LValue callResult = vmCall(
2955                 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
2956                 scope, weakPointer(table));
2957             setJSValue(callResult);
2958             return;
2959         }
2960
2961         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateActivation slow path"));
2962         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateActivation continuation"));
2963
2964         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2965
2966         LValue fastObject = allocateObject<JSLexicalEnvironment>(
2967             JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
2968
2969         // We don't need memory barriers since we just fast-created the activation, so the
2970         // activation must be young.
2971         m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
2972         m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
2973
2974         for (unsigned i = 0; i < table->scopeSize(); ++i) {
2975             m_out.store64(
2976                 m_out.constInt64(JSValue::encode(jsUndefined())),
2977                 fastObject, m_heaps.JSEnvironmentRecord_variables[i]);
2978         }
2979
2980         ValueFromBlock fastResult = m_out.anchor(fastObject);
2981         m_out.jump(continuation);
2982
2983         m_out.appendTo(slowPath, continuation);
2984         LValue callResult = vmCall(
2985             m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
2986             scope, weakPointer(table));
2987         ValueFromBlock slowResult = m_out.anchor(callResult);
2988         m_out.jump(continuation);
2989
2990         m_out.appendTo(continuation, lastNext);
2991         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2992     }
2993
2994     void compileNewFunction()
2995     {
2996         LValue scope = lowCell(m_node->child1());
2997         FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>();
2998         if (executable->singletonFunction()->isStillValid()) {
2999             LValue callResult = vmCall(
3000                 m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable));
3001             setJSValue(callResult);
3002             return;
3003         }
3004
3005         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure();
3006
3007         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewFunction slow path"));
3008         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewFunction continuation"));
3009
3010         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3011
3012         LValue fastObject = allocateObject<JSFunction>(
3013             structure, m_out.intPtrZero, slowPath);
3014
3015         // We don't need memory barriers since we just fast-created the function, so it
3016         // must be young.
3017         m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope);
3018         m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable);
3019         m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData);
3020
3021         ValueFromBlock fastResult = m_out.anchor(fastObject);
3022         m_out.jump(continuation);
3023
3024         m_out.appendTo(slowPath, continuation);
3025         LValue callResult = vmCall(
3026             m_out.operation(operationNewFunctionWithInvalidatedReallocationWatchpoint),
3027             m_callFrame, scope, weakPointer(executable));
3028         ValueFromBlock slowResult = m_out.anchor(callResult);
3029         m_out.jump(continuation);
3030
3031         m_out.appendTo(continuation, lastNext);
3032         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3033     }
3034
3035     void compileCreateDirectArguments()
3036     {
3037         // FIXME: A more effective way of dealing with the argument count and callee is to have
3038         // them be explicit arguments to this node.
3039         // https://bugs.webkit.org/show_bug.cgi?id=142207
3040
3041         Structure* structure =
3042             m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure();
3043
3044         unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1;
3045
3046         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments slow path"));
3047         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments continuation"));
3048
3049         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3050
3051         ArgumentsLength length = getArgumentsLength();
3052
3053         LValue fastObject;
3054         if (length.isKnown) {
3055             fastObject = allocateObject<DirectArguments>(
3056                 DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure,
3057                 m_out.intPtrZero, slowPath);
3058         } else {
3059             LValue size = m_out.add(
3060