Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "DFGOSRAvailabilityAnalysisPhase.h"
  35 #include "DirectArguments.h"
  36 #include "FTLAbstractHeapRepository.h"
  37 #include "FTLAvailableRecovery.h"
  38 #include "FTLForOSREntryJITCode.h"
  39 #include "FTLFormattedValue.h"
  40 #include "FTLInlineCacheSize.h"
  41 #include "FTLLoweredNodeValue.h"
  42 #include "FTLOperations.h"
  43 #include "FTLOutput.h"
  44 #include "FTLThunks.h"
  45 #include "FTLWeightedTarget.h"
  46 #include "JSCInlines.h"
  47 #include "JSLexicalEnvironment.h"
  48 #include "OperandsInlines.h"
  49 #include "ScopedArguments.h"
  50 #include "ScopedArgumentsTable.h"
  51 #include "VirtualRegister.h"
  52 #include <atomic>
  53 #include <dlfcn.h>
  54 #include <llvm/InitializeLLVM.h>
  55 #include <wtf/ProcessID.h>
  56
  57 #if ENABLE(FTL_NATIVE_CALL_INLINING)
  58 #include "BundlePath.h"
  59 #endif
  60
  61 namespace JSC { namespace FTL {
  62
  63 using namespace DFG;
  64
  65 static std::atomic<int> compileCounter;
  66
  67 #if ASSERT_DISABLED
  68 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
  69 {
  70     CRASH();
  71 }
  72 #else
  73 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
  74     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
  75 {
  76     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
  77     if (nodeIndex != UINT_MAX)
  78         dataLog(", node @", nodeIndex);
  79     dataLog(".\n");
  80     CRASH();
  81 }
  82 #endif
  83
  84 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  85 // significantly less dead code.
  86 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  87         FormattedValue _ftc_lowValue = (lowValue);                      \
  88         Edge _ftc_highValue = (highValue);                              \
  89         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  90         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  91             break;                                                      \
  92         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  93     } while (false)
  94
  95 class LowerDFGToLLVM {
  96 public:
  97     LowerDFGToLLVM(State& state)
  98         : m_graph(state.graph)
  99         , m_ftlState(state)
 100         , m_heaps(state.context)
 101         , m_out(state.context)
 102         , m_state(state.graph)
 103         , m_interpreter(state.graph, m_state)
 104         , m_stackmapIDs(0)
 105         , m_tbaaKind(mdKindID(state.context, "tbaa"))
 106         , m_tbaaStructKind(mdKindID(state.context, "tbaa.struct"))
 107     {
 108     }
 109
 110     void lower()
 111     {
 112         CString name;
 113         if (verboseCompilationEnabled()) {
 114             name = toCString(
 115                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
 116                 "_", codeBlock()->hash());
 117         } else
 118             name = "jsBody";
 119
 120         m_graph.m_dominators.computeIfNecessary(m_graph);
 121
 122         m_ftlState.module =
 123             moduleCreateWithNameInContext(name.data(), m_ftlState.context);
 124
 125         m_ftlState.function = addFunction(
 126             m_ftlState.module, name.data(), functionType(m_out.int64));
 127         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
 128         if (isX86() && Options::llvmDisallowAVX()) {
 129             // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
 130             // slower. It should be disabled.
 131             addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
 132         }
 133
 134         if (verboseCompilationEnabled())
 135             dataLog("Function ready, beginning lowering.\n");
 136
 137         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 138
 139         m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
 140         LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
 141         m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
 142
 143         LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments"));
 144
 145         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 146             m_highBlock = m_graph.block(blockIndex);
 147             if (!m_highBlock)
 148                 continue;
 149             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 150         }
 151
 152         m_out.appendTo(m_prologue, stackOverflow);
 153         createPhiVariables();
 154
 155         auto preOrder = m_graph.blocksInPreOrder();
 156
 157         int maxNumberOfArguments = -1;
 158         for (BasicBlock* block : preOrder) {
 159             for (unsigned nodeIndex = block->size(); nodeIndex--; ) {
 160                 Node* node = block->at(nodeIndex);
 161                 switch (node->op()) {
 162                 case NativeCall:
 163                 case NativeConstruct: {
 164                     int numArgs = node->numChildren();
 165                     if (numArgs > maxNumberOfArguments)
 166                         maxNumberOfArguments = numArgs;
 167                     break;
 168                 }
 169                 default:
 170                     break;
 171                 }
 172             }
 173         }
 174
 175         if (maxNumberOfArguments >= 0) {
 176             m_execState = m_out.alloca(arrayType(m_out.int64, JSStack::CallFrameHeaderSize + maxNumberOfArguments));
 177             m_execStorage = m_out.ptrToInt(m_execState, m_out.intPtr);
 178         }
 179
 180         LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
 181
 182         m_captured = m_out.add(
 183             m_out.ptrToInt(capturedAlloca, m_out.intPtr),
 184             m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
 185
 186         m_ftlState.capturedStackmapID = m_stackmapIDs++;
 187         m_out.call(
 188             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
 189             m_out.int32Zero, capturedAlloca);
 190
 191         // If we have any CallVarargs then we nee to have a spill slot for it.
 192         bool hasVarargs = false;
 193         for (BasicBlock* block : preOrder) {
 194             for (Node* node : *block) {
 195                 switch (node->op()) {
 196                 case CallVarargs:
 197                 case CallForwardVarargs:
 198                 case ConstructVarargs:
 199                 case ConstructForwardVarargs:
 200                     hasVarargs = true;
 201                     break;
 202                 default:
 203                     break;
 204                 }
 205             }
 206         }
 207         if (hasVarargs) {
 208             LValue varargsSpillSlots = m_out.alloca(
 209                 arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded()));
 210             m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++;
 211             m_out.call(
 212                 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID),
 213                 m_out.int32Zero, varargsSpillSlots);
 214         }
 215
 216         // We should not create any alloca's after this point, since they will cease to
 217         // be mem2reg candidates.
 218
 219         m_callFrame = m_out.ptrToInt(
 220             m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
 221         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 222         m_tagMask = m_out.constInt64(TagMask);
 223
 224         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 225
 226         m_out.branch(
 227             didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
 228
 229         m_out.appendTo(stackOverflow, m_handleExceptions);
 230         m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 231         m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
 232         m_out.call(
 233             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
 234             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 235         m_out.unreachable();
 236
 237         m_out.appendTo(m_handleExceptions, checkArguments);
 238         m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
 239         m_out.call(
 240             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
 241             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 242         m_out.unreachable();
 243
 244         m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
 245         availabilityMap().clear();
 246         availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
 247         for (unsigned i = codeBlock()->numParameters(); i--;) {
 248             availabilityMap().m_locals.argument(i) =
 249                 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
 250         }
 251         m_codeOriginForExitTarget = CodeOrigin(0);
 252         m_codeOriginForExitProfile = CodeOrigin(0);
 253         m_node = nullptr;
 254         for (unsigned i = codeBlock()->numParameters(); i--;) {
 255             Node* node = m_graph.m_arguments[i];
 256             VirtualRegister operand = virtualRegisterForArgument(i);
 257
 258             LValue jsValue = m_out.load64(addressFor(operand));
 259
 260             if (node) {
 261                 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
 262
 263                 // This is a hack, but it's an effective one. It allows us to do CSE on the
 264                 // primordial load of arguments. This assumes that the GetLocal that got put in
 265                 // place of the original SetArgument doesn't have any effects before it. This
 266                 // should hold true.
 267                 m_loadedArgumentValues.add(node, jsValue);
 268             }
 269
 270             switch (m_graph.m_argumentFormats[i]) {
 271             case FlushedInt32:
 272                 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
 273                 break;
 274             case FlushedBoolean:
 275                 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
 276                 break;
 277             case FlushedCell:
 278                 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
 279                 break;
 280             case FlushedJSValue:
 281                 break;
 282             default:
 283                 DFG_CRASH(m_graph, node, "Bad flush format for argument");
 284                 break;
 285             }
 286         }
 287         m_out.jump(lowBlock(m_graph.block(0)));
 288
 289         for (BasicBlock* block : preOrder)
 290             compileBlock(block);
 291
 292         if (Options::dumpLLVMIR())
 293             dumpModule(m_ftlState.module);
 294
 295         if (verboseCompilationEnabled())
 296             m_ftlState.dumpState("after lowering");
 297         if (validationEnabled())
 298             verifyModule(m_ftlState.module);
 299     }
 300
 301 private:
 302
 303     void createPhiVariables()
 304     {
 305         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 306             BasicBlock* block = m_graph.block(blockIndex);
 307             if (!block)
 308                 continue;
 309             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 310                 Node* node = block->at(nodeIndex);
 311                 if (node->op() != Phi)
 312                     continue;
 313                 LType type;
 314                 switch (node->flags() & NodeResultMask) {
 315                 case NodeResultDouble:
 316                     type = m_out.doubleType;
 317                     break;
 318                 case NodeResultInt32:
 319                     type = m_out.int32;
 320                     break;
 321                 case NodeResultInt52:
 322                     type = m_out.int64;
 323                     break;
 324                 case NodeResultBoolean:
 325                     type = m_out.boolean;
 326                     break;
 327                 case NodeResultJS:
 328                     type = m_out.int64;
 329                     break;
 330                 default:
 331                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
 332                     break;
 333                 }
 334                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 335             }
 336         }
 337     }
 338
 339     void compileBlock(BasicBlock* block)
 340     {
 341         if (!block)
 342             return;
 343
 344         if (verboseCompilationEnabled())
 345             dataLog("Compiling block ", *block, "\n");
 346
 347         m_highBlock = block;
 348
 349         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 350
 351         m_nextHighBlock = 0;
 352         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 353             m_nextHighBlock = m_graph.block(nextBlockIndex);
 354             if (m_nextHighBlock)
 355                 break;
 356         }
 357         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 358
 359         // All of this effort to find the next block gives us the ability to keep the
 360         // generated IR in roughly program order. This ought not affect the performance
 361         // of the generated code (since we expect LLVM to reorder things) but it will
 362         // make IR dumps easier to read.
 363         m_out.appendTo(lowBlock, m_nextLowBlock);
 364
 365         if (Options::ftlCrashes())
 366             m_out.trap();
 367
 368         if (!m_highBlock->cfaHasVisited) {
 369             if (verboseCompilationEnabled())
 370                 dataLog("Bailing because CFA didn't reach.\n");
 371             crash(m_highBlock->index, UINT_MAX);
 372             return;
 373         }
 374
 375         m_availabilityCalculator.beginBlock(m_highBlock);
 376
 377         m_state.reset();
 378         m_state.beginBasicBlock(m_highBlock);
 379
 380         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 381             if (!compileNode(m_nodeIndex))
 382                 break;
 383         }
 384     }
 385
 386     void safelyInvalidateAfterTermination()
 387     {
 388         if (verboseCompilationEnabled())
 389             dataLog("Bailing.\n");
 390         crash();
 391
 392         // Invalidate dominated blocks. Under normal circumstances we would expect
 393         // them to be invalidated already. But you can have the CFA become more
 394         // precise over time because the structures of objects change on the main
 395         // thread. Failing to do this would result in weird crashes due to a value
 396         // being used but not defined. Race conditions FTW!
 397         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 398             BasicBlock* target = m_graph.block(blockIndex);
 399             if (!target)
 400                 continue;
 401             if (m_graph.m_dominators.dominates(m_highBlock, target)) {
 402                 if (verboseCompilationEnabled())
 403                     dataLog("Block ", *target, " will bail also.\n");
 404                 target->cfaHasVisited = false;
 405             }
 406         }
 407     }
 408
 409     bool compileNode(unsigned nodeIndex)
 410     {
 411         if (!m_state.isValid()) {
 412             safelyInvalidateAfterTermination();
 413             return false;
 414         }
 415
 416         m_node = m_highBlock->at(nodeIndex);
 417         m_codeOriginForExitProfile = m_node->origin.semantic;
 418         m_codeOriginForExitTarget = m_node->origin.forExit;
 419
 420         if (verboseCompilationEnabled())
 421             dataLog("Lowering ", m_node, "\n");
 422
 423         m_availableRecoveries.resize(0);
 424
 425         m_interpreter.startExecuting();
 426
 427         switch (m_node->op()) {
 428         case Upsilon:
 429             compileUpsilon();
 430             break;
 431         case Phi:
 432             compilePhi();
 433             break;
 434         case JSConstant:
 435             break;
 436         case DoubleConstant:
 437             compileDoubleConstant();
 438             break;
 439         case Int52Constant:
 440             compileInt52Constant();
 441             break;
 442         case DoubleRep:
 443             compileDoubleRep();
 444             break;
 445         case ValueRep:
 446             compileValueRep();
 447             break;
 448         case Int52Rep:
 449             compileInt52Rep();
 450             break;
 451         case ValueToInt32:
 452             compileValueToInt32();
 453             break;
 454         case BooleanToNumber:
 455             compileBooleanToNumber();
 456             break;
 457         case ExtractOSREntryLocal:
 458             compileExtractOSREntryLocal();
 459             break;
 460         case GetStack:
 461             compileGetStack();
 462             break;
 463         case PutStack:
 464             compilePutStack();
 465             break;
 466         case Phantom:
 467         case Check:
 468             compilePhantom();
 469             break;
 470         case ToThis:
 471             compileToThis();
 472             break;
 473         case ValueAdd:
 474             compileValueAdd();
 475             break;
 476         case ArithAdd:
 477         case ArithSub:
 478             compileArithAddOrSub();
 479             break;
 480         case ArithClz32:
 481             compileArithClz32();
 482             break;
 483         case ArithMul:
 484             compileArithMul();
 485             break;
 486         case ArithDiv:
 487             compileArithDiv();
 488             break;
 489         case ArithMod:
 490             compileArithMod();
 491             break;
 492         case ArithMin:
 493         case ArithMax:
 494             compileArithMinOrMax();
 495             break;
 496         case ArithAbs:
 497             compileArithAbs();
 498             break;
 499         case ArithSin:
 500             compileArithSin();
 501             break;
 502         case ArithCos:
 503             compileArithCos();
 504             break;
 505         case ArithPow:
 506             compileArithPow();
 507             break;
 508         case ArithSqrt:
 509             compileArithSqrt();
 510             break;
 511         case ArithLog:
 512             compileArithLog();
 513             break;
 514         case ArithFRound:
 515             compileArithFRound();
 516             break;
 517         case ArithNegate:
 518             compileArithNegate();
 519             break;
 520         case BitAnd:
 521             compileBitAnd();
 522             break;
 523         case BitOr:
 524             compileBitOr();
 525             break;
 526         case BitXor:
 527             compileBitXor();
 528             break;
 529         case BitRShift:
 530             compileBitRShift();
 531             break;
 532         case BitLShift:
 533             compileBitLShift();
 534             break;
 535         case BitURShift:
 536             compileBitURShift();
 537             break;
 538         case UInt32ToNumber:
 539             compileUInt32ToNumber();
 540             break;
 541         case CheckStructure:
 542             compileCheckStructure();
 543             break;
 544         case CheckCell:
 545             compileCheckCell();
 546             break;
 547         case CheckNotEmpty:
 548             compileCheckNotEmpty();
 549             break;
 550         case CheckBadCell:
 551             compileCheckBadCell();
 552             break;
 553         case GetExecutable:
 554             compileGetExecutable();
 555             break;
 556         case ArrayifyToStructure:
 557             compileArrayifyToStructure();
 558             break;
 559         case PutStructure:
 560             compilePutStructure();
 561             break;
 562         case GetById:
 563             compileGetById();
 564             break;
 565         case In:
 566             compileIn();
 567             break;
 568         case PutById:
 569         case PutByIdDirect:
 570             compilePutById();
 571             break;
 572         case GetButterfly:
 573             compileGetButterfly();
 574             break;
 575         case ConstantStoragePointer:
 576             compileConstantStoragePointer();
 577             break;
 578         case GetIndexedPropertyStorage:
 579             compileGetIndexedPropertyStorage();
 580             break;
 581         case CheckArray:
 582             compileCheckArray();
 583             break;
 584         case GetArrayLength:
 585             compileGetArrayLength();
 586             break;
 587         case CheckInBounds:
 588             compileCheckInBounds();
 589             break;
 590         case GetByVal:
 591             compileGetByVal();
 592             break;
 593         case GetMyArgumentByVal:
 594             compileGetMyArgumentByVal();
 595             break;
 596         case PutByVal:
 597         case PutByValAlias:
 598         case PutByValDirect:
 599             compilePutByVal();
 600             break;
 601         case ArrayPush:
 602             compileArrayPush();
 603             break;
 604         case ArrayPop:
 605             compileArrayPop();
 606             break;
 607         case CreateActivation:
 608             compileCreateActivation();
 609             break;
 610         case NewFunction:
 611             compileNewFunction();
 612             break;
 613         case CreateDirectArguments:
 614             compileCreateDirectArguments();
 615             break;
 616         case CreateScopedArguments:
 617             compileCreateScopedArguments();
 618             break;
 619         case CreateClonedArguments:
 620             compileCreateClonedArguments();
 621             break;
 622         case NewObject:
 623             compileNewObject();
 624             break;
 625         case NewArray:
 626             compileNewArray();
 627             break;
 628         case NewArrayBuffer:
 629             compileNewArrayBuffer();
 630             break;
 631         case NewArrayWithSize:
 632             compileNewArrayWithSize();
 633             break;
 634         case GetTypedArrayByteOffset:
 635             compileGetTypedArrayByteOffset();
 636             break;
 637         case AllocatePropertyStorage:
 638             compileAllocatePropertyStorage();
 639             break;
 640         case ReallocatePropertyStorage:
 641             compileReallocatePropertyStorage();
 642             break;
 643         case ToString:
 644         case CallStringConstructor:
 645             compileToStringOrCallStringConstructor();
 646             break;
 647         case ToPrimitive:
 648             compileToPrimitive();
 649             break;
 650         case MakeRope:
 651             compileMakeRope();
 652             break;
 653         case StringCharAt:
 654             compileStringCharAt();
 655             break;
 656         case StringCharCodeAt:
 657             compileStringCharCodeAt();
 658             break;
 659         case GetByOffset:
 660         case GetGetterSetterByOffset:
 661             compileGetByOffset();
 662             break;
 663         case GetGetter:
 664             compileGetGetter();
 665             break;
 666         case GetSetter:
 667             compileGetSetter();
 668             break;
 669         case MultiGetByOffset:
 670             compileMultiGetByOffset();
 671             break;
 672         case PutByOffset:
 673             compilePutByOffset();
 674             break;
 675         case MultiPutByOffset:
 676             compileMultiPutByOffset();
 677             break;
 678         case GetGlobalVar:
 679             compileGetGlobalVar();
 680             break;
 681         case PutGlobalVar:
 682             compilePutGlobalVar();
 683             break;
 684         case NotifyWrite:
 685             compileNotifyWrite();
 686             break;
 687         case GetCallee:
 688             compileGetCallee();
 689             break;
 690         case GetArgumentCount:
 691             compileGetArgumentCount();
 692             break;
 693         case GetScope:
 694             compileGetScope();
 695             break;
 696         case SkipScope:
 697             compileSkipScope();
 698             break;
 699         case GetClosureVar:
 700             compileGetClosureVar();
 701             break;
 702         case PutClosureVar:
 703             compilePutClosureVar();
 704             break;
 705         case GetFromArguments:
 706             compileGetFromArguments();
 707             break;
 708         case PutToArguments:
 709             compilePutToArguments();
 710             break;
 711         case CompareEq:
 712             compileCompareEq();
 713             break;
 714         case CompareEqConstant:
 715             compileCompareEqConstant();
 716             break;
 717         case CompareStrictEq:
 718             compileCompareStrictEq();
 719             break;
 720         case CompareLess:
 721             compileCompareLess();
 722             break;
 723         case CompareLessEq:
 724             compileCompareLessEq();
 725             break;
 726         case CompareGreater:
 727             compileCompareGreater();
 728             break;
 729         case CompareGreaterEq:
 730             compileCompareGreaterEq();
 731             break;
 732         case LogicalNot:
 733             compileLogicalNot();
 734             break;
 735         case Call:
 736         case Construct:
 737             compileCallOrConstruct();
 738             break;
 739         case CallVarargs:
 740         case CallForwardVarargs:
 741         case ConstructVarargs:
 742         case ConstructForwardVarargs:
 743             compileCallOrConstructVarargs();
 744             break;
 745         case LoadVarargs:
 746             compileLoadVarargs();
 747             break;
 748         case ForwardVarargs:
 749             compileForwardVarargs();
 750             break;
 751 #if ENABLE(FTL_NATIVE_CALL_INLINING)
 752         case NativeCall:
 753         case NativeConstruct:
 754             compileNativeCallOrConstruct();
 755             break;
 756 #endif
 757         case Jump:
 758             compileJump();
 759             break;
 760         case Branch:
 761             compileBranch();
 762             break;
 763         case Switch:
 764             compileSwitch();
 765             break;
 766         case Return:
 767             compileReturn();
 768             break;
 769         case ForceOSRExit:
 770             compileForceOSRExit();
 771             break;
 772         case Throw:
 773         case ThrowReferenceError:
 774             compileThrow();
 775             break;
 776         case InvalidationPoint:
 777             compileInvalidationPoint();
 778             break;
 779         case IsUndefined:
 780             compileIsUndefined();
 781             break;
 782         case IsBoolean:
 783             compileIsBoolean();
 784             break;
 785         case IsNumber:
 786             compileIsNumber();
 787             break;
 788         case IsString:
 789             compileIsString();
 790             break;
 791         case IsObject:
 792             compileIsObject();
 793             break;
 794         case IsObjectOrNull:
 795             compileIsObjectOrNull();
 796             break;
 797         case IsFunction:
 798             compileIsFunction();
 799             break;
 800         case CheckHasInstance:
 801             compileCheckHasInstance();
 802             break;
 803         case InstanceOf:
 804             compileInstanceOf();
 805             break;
 806         case CountExecution:
 807             compileCountExecution();
 808             break;
 809         case StoreBarrier:
 810             compileStoreBarrier();
 811             break;
 812         case StoreBarrierWithNullCheck:
 813             compileStoreBarrierWithNullCheck();
 814             break;
 815         case HasIndexedProperty:
 816             compileHasIndexedProperty();
 817             break;
 818         case HasGenericProperty:
 819             compileHasGenericProperty();
 820             break;
 821         case HasStructureProperty:
 822             compileHasStructureProperty();
 823             break;
 824         case GetDirectPname:
 825             compileGetDirectPname();
 826             break;
 827         case GetEnumerableLength:
 828             compileGetEnumerableLength();
 829             break;
 830         case GetPropertyEnumerator:
 831             compileGetPropertyEnumerator();
 832             break;
 833         case GetEnumeratorStructurePname:
 834             compileGetEnumeratorStructurePname();
 835             break;
 836         case GetEnumeratorGenericPname:
 837             compileGetEnumeratorGenericPname();
 838             break;
 839         case ToIndexString:
 840             compileToIndexString();
 841             break;
 842         case CheckStructureImmediate:
 843             compileCheckStructureImmediate();
 844             break;
 845         case MaterializeNewObject:
 846             compileMaterializeNewObject();
 847             break;
 848
 849         case PhantomLocal:
 850         case LoopHint:
 851         case MovHint:
 852         case ZombieHint:
 853         case PhantomNewObject:
 854         case PhantomNewFunction:
 855         case PhantomDirectArguments:
 856         case PhantomClonedArguments:
 857         case PutHint:
 858         case BottomValue:
 859         case KillStack:
 860             break;
 861         default:
 862             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
 863             break;
 864         }
 865
 866         if (m_node->isTerminal())
 867             return false;
 868
 869         if (!m_state.isValid()) {
 870             safelyInvalidateAfterTermination();
 871             return false;
 872         }
 873
 874         m_availabilityCalculator.executeNode(m_node);
 875         m_interpreter.executeEffects(nodeIndex);
 876
 877         return true;
 878     }
 879
 880     void compileUpsilon()
 881     {
 882         LValue destination = m_phis.get(m_node->phi());
 883
 884         switch (m_node->child1().useKind()) {
 885         case DoubleRepUse:
 886             m_out.set(lowDouble(m_node->child1()), destination);
 887             break;
 888         case Int32Use:
 889             m_out.set(lowInt32(m_node->child1()), destination);
 890             break;
 891         case Int52RepUse:
 892             m_out.set(lowInt52(m_node->child1()), destination);
 893             break;
 894         case BooleanUse:
 895             m_out.set(lowBoolean(m_node->child1()), destination);
 896             break;
 897         case CellUse:
 898             m_out.set(lowCell(m_node->child1()), destination);
 899             break;
 900         case UntypedUse:
 901             m_out.set(lowJSValue(m_node->child1()), destination);
 902             break;
 903         default:
 904             DFG_CRASH(m_graph, m_node, "Bad use kind");
 905             break;
 906         }
 907     }
 908
 909     void compilePhi()
 910     {
 911         LValue source = m_phis.get(m_node);
 912
 913         switch (m_node->flags() & NodeResultMask) {
 914         case NodeResultDouble:
 915             setDouble(m_out.get(source));
 916             break;
 917         case NodeResultInt32:
 918             setInt32(m_out.get(source));
 919             break;
 920         case NodeResultInt52:
 921             setInt52(m_out.get(source));
 922             break;
 923         case NodeResultBoolean:
 924             setBoolean(m_out.get(source));
 925             break;
 926         case NodeResultJS:
 927             setJSValue(m_out.get(source));
 928             break;
 929         default:
 930             DFG_CRASH(m_graph, m_node, "Bad use kind");
 931             break;
 932         }
 933     }
 934
 935     void compileDoubleConstant()
 936     {
 937         setDouble(m_out.constDouble(m_node->asNumber()));
 938     }
 939
 940     void compileInt52Constant()
 941     {
 942         int64_t value = m_node->asMachineInt();
 943
 944         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
 945         setStrictInt52(m_out.constInt64(value));
 946     }
 947
 948     void compileDoubleRep()
 949     {
 950         switch (m_node->child1().useKind()) {
 951         case NumberUse: {
 952             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 953             setDouble(jsValueToDouble(m_node->child1(), value));
 954             return;
 955         }
 956
 957         case Int52RepUse: {
 958             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
 959             return;
 960         }
 961
 962         default:
 963             DFG_CRASH(m_graph, m_node, "Bad use kind");
 964         }
 965     }
 966
 967     void compileValueRep()
 968     {
 969         switch (m_node->child1().useKind()) {
 970         case DoubleRepUse: {
 971             LValue value = lowDouble(m_node->child1());
 972
 973             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
 974                 value = m_out.select(
 975                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
 976             }
 977
 978             setJSValue(boxDouble(value));
 979             return;
 980         }
 981
 982         case Int52RepUse: {
 983             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
 984             return;
 985         }
 986
 987         default:
 988             DFG_CRASH(m_graph, m_node, "Bad use kind");
 989         }
 990     }
 991
 992     void compileInt52Rep()
 993     {
 994         switch (m_node->child1().useKind()) {
 995         case Int32Use:
 996             setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
 997             return;
 998
 999         case MachineIntUse:
1000             setStrictInt52(
1001                 jsValueToStrictInt52(
1002                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1003             return;
1004
1005         case DoubleRepMachineIntUse:
1006             setStrictInt52(
1007                 doubleToStrictInt52(
1008                     m_node->child1(), lowDouble(m_node->child1())));
1009             return;
1010
1011         default:
1012             RELEASE_ASSERT_NOT_REACHED();
1013         }
1014     }
1015
1016     void compileValueToInt32()
1017     {
1018         switch (m_node->child1().useKind()) {
1019         case Int52RepUse:
1020             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1021             break;
1022
1023         case DoubleRepUse:
1024             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1025             break;
1026
1027         case NumberUse:
1028         case NotCellUse: {
1029             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1030             if (isValid(value)) {
1031                 setInt32(value.value());
1032                 break;
1033             }
1034
1035             value = m_jsValueValues.get(m_node->child1().node());
1036             if (isValid(value)) {
1037                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1038                 break;
1039             }
1040
1041             // We'll basically just get here for constants. But it's good to have this
1042             // catch-all since we often add new representations into the mix.
1043             setInt32(
1044                 numberOrNotCellToInt32(
1045                     m_node->child1(),
1046                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1047             break;
1048         }
1049
1050         default:
1051             DFG_CRASH(m_graph, m_node, "Bad use kind");
1052             break;
1053         }
1054     }
1055
1056     void compileBooleanToNumber()
1057     {
1058         switch (m_node->child1().useKind()) {
1059         case BooleanUse: {
1060             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1061             return;
1062         }
1063
1064         case UntypedUse: {
1065             LValue value = lowJSValue(m_node->child1());
1066
1067             LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
1068             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
1069
1070             ValueFromBlock notBooleanResult = m_out.anchor(value);
1071             m_out.branch(isBoolean(value), unsure(booleanCase), unsure(continuation));
1072
1073             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1074             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1075                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1076             m_out.jump(continuation);
1077
1078             m_out.appendTo(continuation, lastNext);
1079             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1080             return;
1081         }
1082
1083         default:
1084             RELEASE_ASSERT_NOT_REACHED();
1085             return;
1086         }
1087     }
1088
1089     void compileExtractOSREntryLocal()
1090     {
1091         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1092             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1093         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1094     }
1095
1096     void compileGetStack()
1097     {
1098         // GetLocals arise only for captured variables and arguments. For arguments, we might have
1099         // already loaded it.
1100         if (LValue value = m_loadedArgumentValues.get(m_node)) {
1101             setJSValue(value);
1102             return;
1103         }
1104
1105         StackAccessData* data = m_node->stackAccessData();
1106         AbstractValue& value = m_state.variables().operand(data->local);
1107
1108         DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1109         DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1110
1111         if (isInt32Speculation(value.m_type))
1112             setInt32(m_out.load32(payloadFor(data->machineLocal)));
1113         else
1114             setJSValue(m_out.load64(addressFor(data->machineLocal)));
1115     }
1116
1117     void compilePutStack()
1118     {
1119         StackAccessData* data = m_node->stackAccessData();
1120         switch (data->format) {
1121         case FlushedJSValue: {
1122             LValue value = lowJSValue(m_node->child1());
1123             m_out.store64(value, addressFor(data->machineLocal));
1124             break;
1125         }
1126
1127         case FlushedDouble: {
1128             LValue value = lowDouble(m_node->child1());
1129             m_out.storeDouble(value, addressFor(data->machineLocal));
1130             break;
1131         }
1132
1133         case FlushedInt32: {
1134             LValue value = lowInt32(m_node->child1());
1135             m_out.store32(value, payloadFor(data->machineLocal));
1136             break;
1137         }
1138
1139         case FlushedInt52: {
1140             LValue value = lowInt52(m_node->child1());
1141             m_out.store64(value, addressFor(data->machineLocal));
1142             break;
1143         }
1144
1145         case FlushedCell: {
1146             LValue value = lowCell(m_node->child1());
1147             m_out.store64(value, addressFor(data->machineLocal));
1148             break;
1149         }
1150
1151         case FlushedBoolean: {
1152             speculateBoolean(m_node->child1());
1153             m_out.store64(
1154                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1155                 addressFor(data->machineLocal));
1156             break;
1157         }
1158
1159         default:
1160             DFG_CRASH(m_graph, m_node, "Bad flush format");
1161             break;
1162         }
1163     }
1164
1165     void compilePhantom()
1166     {
1167         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1168     }
1169
1170     void compileToThis()
1171     {
1172         LValue value = lowJSValue(m_node->child1());
1173
1174         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
1175         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
1176         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
1177
1178         m_out.branch(isCell(value), usually(isCellCase), rarely(slowCase));
1179
1180         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1181         ValueFromBlock fastResult = m_out.anchor(value);
1182         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1183
1184         m_out.appendTo(slowCase, continuation);
1185         J_JITOperation_EJ function;
1186         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1187             function = operationToThisStrict;
1188         else
1189             function = operationToThis;
1190         ValueFromBlock slowResult = m_out.anchor(
1191             vmCall(m_out.operation(function), m_callFrame, value));
1192         m_out.jump(continuation);
1193
1194         m_out.appendTo(continuation, lastNext);
1195         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1196     }
1197
1198     void compileValueAdd()
1199     {
1200         J_JITOperation_EJJ operation;
1201         if (!(m_state.forNode(m_node->child1()).m_type & SpecFullNumber)
1202             && !(m_state.forNode(m_node->child2()).m_type & SpecFullNumber))
1203             operation = operationValueAddNotNumber;
1204         else
1205             operation = operationValueAdd;
1206         setJSValue(vmCall(
1207             m_out.operation(operation), m_callFrame,
1208             lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1209     }
1210
1211     void compileArithAddOrSub()
1212     {
1213         bool isSub =  m_node->op() == ArithSub;
1214         switch (m_node->binaryUseKind()) {
1215         case Int32Use: {
1216             LValue left = lowInt32(m_node->child1());
1217             LValue right = lowInt32(m_node->child2());
1218
1219             if (!shouldCheckOverflow(m_node->arithMode())) {
1220                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1221                 break;
1222             }
1223
1224             LValue result;
1225             if (!isSub) {
1226                 result = m_out.addWithOverflow32(left, right);
1227
1228                 if (doesKill(m_node->child2())) {
1229                     addAvailableRecovery(
1230                         m_node->child2(), SubRecovery,
1231                         m_out.extractValue(result, 0), left, ValueFormatInt32);
1232                 } else if (doesKill(m_node->child1())) {
1233                     addAvailableRecovery(
1234                         m_node->child1(), SubRecovery,
1235                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1236                 }
1237             } else {
1238                 result = m_out.subWithOverflow32(left, right);
1239
1240                 if (doesKill(m_node->child2())) {
1241                     // result = left - right
1242                     // result - left = -right
1243                     // right = left - result
1244                     addAvailableRecovery(
1245                         m_node->child2(), SubRecovery,
1246                         left, m_out.extractValue(result, 0), ValueFormatInt32);
1247                 } else if (doesKill(m_node->child1())) {
1248                     // result = left - right
1249                     // result + right = left
1250                     addAvailableRecovery(
1251                         m_node->child1(), AddRecovery,
1252                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1253                 }
1254             }
1255
1256             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1257             setInt32(m_out.extractValue(result, 0));
1258             break;
1259         }
1260
1261         case Int52RepUse: {
1262             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)
1263                 && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) {
1264                 Int52Kind kind;
1265                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1266                 LValue right = lowInt52(m_node->child2(), kind);
1267                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1268                 break;
1269             }
1270
1271             LValue left = lowInt52(m_node->child1());
1272             LValue right = lowInt52(m_node->child2());
1273
1274             LValue result;
1275             if (!isSub) {
1276                 result = m_out.addWithOverflow64(left, right);
1277
1278                 if (doesKill(m_node->child2())) {
1279                     addAvailableRecovery(
1280                         m_node->child2(), SubRecovery,
1281                         m_out.extractValue(result, 0), left, ValueFormatInt52);
1282                 } else if (doesKill(m_node->child1())) {
1283                     addAvailableRecovery(
1284                         m_node->child1(), SubRecovery,
1285                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1286                 }
1287             } else {
1288                 result = m_out.subWithOverflow64(left, right);
1289
1290                 if (doesKill(m_node->child2())) {
1291                     // result = left - right
1292                     // result - left = -right
1293                     // right = left - result
1294                     addAvailableRecovery(
1295                         m_node->child2(), SubRecovery,
1296                         left, m_out.extractValue(result, 0), ValueFormatInt52);
1297                 } else if (doesKill(m_node->child1())) {
1298                     // result = left - right
1299                     // result + right = left
1300                     addAvailableRecovery(
1301                         m_node->child1(), AddRecovery,
1302                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1303                 }
1304             }
1305
1306             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1307             setInt52(m_out.extractValue(result, 0));
1308             break;
1309         }
1310
1311         case DoubleRepUse: {
1312             LValue C1 = lowDouble(m_node->child1());
1313             LValue C2 = lowDouble(m_node->child2());
1314
1315             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1316             break;
1317         }
1318
1319         default:
1320             DFG_CRASH(m_graph, m_node, "Bad use kind");
1321             break;
1322         }
1323     }
1324
1325     void compileArithClz32()
1326     {
1327         LValue operand = lowInt32(m_node->child1());
1328         LValue isZeroUndef = m_out.booleanFalse;
1329         setInt32(m_out.ctlz32(operand, isZeroUndef));
1330     }
1331
1332     void compileArithMul()
1333     {
1334         switch (m_node->binaryUseKind()) {
1335         case Int32Use: {
1336             LValue left = lowInt32(m_node->child1());
1337             LValue right = lowInt32(m_node->child2());
1338
1339             LValue result;
1340
1341             if (!shouldCheckOverflow(m_node->arithMode()))
1342                 result = m_out.mul(left, right);
1343             else {
1344                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1345                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1346                 result = m_out.extractValue(overflowResult, 0);
1347             }
1348
1349             if (shouldCheckNegativeZero(m_node->arithMode())) {
1350                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1351                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1352
1353                 m_out.branch(
1354                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1355
1356                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1357                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1358                 speculate(NegativeZero, noValue(), 0, cond);
1359                 m_out.jump(continuation);
1360                 m_out.appendTo(continuation, lastNext);
1361             }
1362
1363             setInt32(result);
1364             break;
1365         }
1366
1367         case Int52RepUse: {
1368             Int52Kind kind;
1369             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1370             LValue right = lowInt52(m_node->child2(), opposite(kind));
1371
1372             LValue overflowResult = m_out.mulWithOverflow64(left, right);
1373             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1374             LValue result = m_out.extractValue(overflowResult, 0);
1375
1376             if (shouldCheckNegativeZero(m_node->arithMode())) {
1377                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1378                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1379
1380                 m_out.branch(
1381                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1382
1383                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1384                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1385                 speculate(NegativeZero, noValue(), 0, cond);
1386                 m_out.jump(continuation);
1387                 m_out.appendTo(continuation, lastNext);
1388             }
1389
1390             setInt52(result);
1391             break;
1392         }
1393
1394         case DoubleRepUse: {
1395             setDouble(
1396                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1397             break;
1398         }
1399
1400         default:
1401             DFG_CRASH(m_graph, m_node, "Bad use kind");
1402             break;
1403         }
1404     }
1405
1406     void compileArithDiv()
1407     {
1408         switch (m_node->binaryUseKind()) {
1409         case Int32Use: {
1410             LValue numerator = lowInt32(m_node->child1());
1411             LValue denominator = lowInt32(m_node->child2());
1412
1413             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1414             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1415             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1416
1417             Vector<ValueFromBlock, 3> results;
1418
1419             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1420
1421             m_out.branch(
1422                 m_out.above(adjustedDenominator, m_out.int32One),
1423                 usually(continuation), rarely(unsafeDenominator));
1424
1425             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1426
1427             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1428
1429             if (shouldCheckOverflow(m_node->arithMode())) {
1430                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1431                 speculate(Overflow, noValue(), 0, cond);
1432                 m_out.jump(continuation);
1433             } else {
1434                 // This is the case where we convert the result to an int after we're done. So,
1435                 // if the denominator is zero, then the result should be zero.
1436                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1437                 // check above) and the numerator is -2^31 then the result should be -2^31.
1438
1439                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1440                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1441                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1442
1443                 m_out.branch(
1444                     m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1445
1446                 m_out.appendTo(divByZero, notDivByZero);
1447                 results.append(m_out.anchor(m_out.int32Zero));
1448                 m_out.jump(done);
1449
1450                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1451                 m_out.branch(
1452                     m_out.equal(numerator, neg2ToThe31),
1453                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1454
1455                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1456                 results.append(m_out.anchor(neg2ToThe31));
1457                 m_out.jump(done);
1458             }
1459
1460             m_out.appendTo(continuation, done);
1461
1462             if (shouldCheckNegativeZero(m_node->arithMode())) {
1463                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1464                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1465
1466                 m_out.branch(
1467                     m_out.isZero32(numerator),
1468                     rarely(zeroNumerator), usually(numeratorContinuation));
1469
1470                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1471
1472                 speculate(
1473                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1474
1475                 m_out.jump(numeratorContinuation);
1476
1477                 m_out.appendTo(numeratorContinuation, innerLastNext);
1478             }
1479
1480             LValue result = m_out.div(numerator, denominator);
1481
1482             if (shouldCheckOverflow(m_node->arithMode())) {
1483                 speculate(
1484                     Overflow, noValue(), 0,
1485                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1486             }
1487
1488             results.append(m_out.anchor(result));
1489             m_out.jump(done);
1490
1491             m_out.appendTo(done, lastNext);
1492
1493             setInt32(m_out.phi(m_out.int32, results));
1494             break;
1495         }
1496
1497         case DoubleRepUse: {
1498             setDouble(m_out.doubleDiv(
1499                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1500             break;
1501         }
1502
1503         default:
1504             DFG_CRASH(m_graph, m_node, "Bad use kind");
1505             break;
1506         }
1507     }
1508
1509     void compileArithMod()
1510     {
1511         switch (m_node->binaryUseKind()) {
1512         case Int32Use: {
1513             LValue numerator = lowInt32(m_node->child1());
1514             LValue denominator = lowInt32(m_node->child2());
1515
1516             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1517             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1518             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1519
1520             Vector<ValueFromBlock, 3> results;
1521
1522             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1523
1524             m_out.branch(
1525                 m_out.above(adjustedDenominator, m_out.int32One),
1526                 usually(continuation), rarely(unsafeDenominator));
1527
1528             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1529
1530             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1531
1532             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1533             // separate case for that. But it probably doesn't matter so much.
1534             if (shouldCheckOverflow(m_node->arithMode())) {
1535                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1536                 speculate(Overflow, noValue(), 0, cond);
1537                 m_out.jump(continuation);
1538             } else {
1539                 // This is the case where we convert the result to an int after we're done. So,
1540                 // if the denominator is zero, then the result should be result should be zero.
1541                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1542                 // check above) and the numerator is -2^31 then the result should be -2^31.
1543
1544                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1545                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1546                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1547
1548                 m_out.branch(
1549                     m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1550
1551                 m_out.appendTo(modByZero, notModByZero);
1552                 results.append(m_out.anchor(m_out.int32Zero));
1553                 m_out.jump(done);
1554
1555                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1556                 m_out.branch(
1557                     m_out.equal(numerator, neg2ToThe31),
1558                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1559
1560                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1561                 results.append(m_out.anchor(m_out.int32Zero));
1562                 m_out.jump(done);
1563             }
1564
1565             m_out.appendTo(continuation, done);
1566
1567             LValue remainder = m_out.rem(numerator, denominator);
1568
1569             if (shouldCheckNegativeZero(m_node->arithMode())) {
1570                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1571                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1572
1573                 m_out.branch(
1574                     m_out.lessThan(numerator, m_out.int32Zero),
1575                     unsure(negativeNumerator), unsure(numeratorContinuation));
1576
1577                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1578
1579                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1580
1581                 m_out.jump(numeratorContinuation);
1582
1583                 m_out.appendTo(numeratorContinuation, innerLastNext);
1584             }
1585
1586             results.append(m_out.anchor(remainder));
1587             m_out.jump(done);
1588
1589             m_out.appendTo(done, lastNext);
1590
1591             setInt32(m_out.phi(m_out.int32, results));
1592             break;
1593         }
1594
1595         case DoubleRepUse: {
1596             setDouble(
1597                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1598             break;
1599         }
1600
1601         default:
1602             DFG_CRASH(m_graph, m_node, "Bad use kind");
1603             break;
1604         }
1605     }
1606
1607     void compileArithMinOrMax()
1608     {
1609         switch (m_node->binaryUseKind()) {
1610         case Int32Use: {
1611             LValue left = lowInt32(m_node->child1());
1612             LValue right = lowInt32(m_node->child2());
1613
1614             setInt32(
1615                 m_out.select(
1616                     m_node->op() == ArithMin
1617                         ? m_out.lessThan(left, right)
1618                         : m_out.lessThan(right, left),
1619                     left, right));
1620             break;
1621         }
1622
1623         case DoubleRepUse: {
1624             LValue left = lowDouble(m_node->child1());
1625             LValue right = lowDouble(m_node->child2());
1626
1627             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1628             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1629
1630             Vector<ValueFromBlock, 2> results;
1631
1632             results.append(m_out.anchor(left));
1633             m_out.branch(
1634                 m_node->op() == ArithMin
1635                     ? m_out.doubleLessThan(left, right)
1636                     : m_out.doubleGreaterThan(left, right),
1637                 unsure(continuation), unsure(notLessThan));
1638
1639             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1640             results.append(m_out.anchor(m_out.select(
1641                 m_node->op() == ArithMin
1642                     ? m_out.doubleGreaterThanOrEqual(left, right)
1643                     : m_out.doubleLessThanOrEqual(left, right),
1644                 right, m_out.constDouble(PNaN))));
1645             m_out.jump(continuation);
1646
1647             m_out.appendTo(continuation, lastNext);
1648             setDouble(m_out.phi(m_out.doubleType, results));
1649             break;
1650         }
1651
1652         default:
1653             DFG_CRASH(m_graph, m_node, "Bad use kind");
1654             break;
1655         }
1656     }
1657
1658     void compileArithAbs()
1659     {
1660         switch (m_node->child1().useKind()) {
1661         case Int32Use: {
1662             LValue value = lowInt32(m_node->child1());
1663
1664             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1665             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1666
1667             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1668
1669             setInt32(result);
1670             break;
1671         }
1672
1673         case DoubleRepUse: {
1674             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1675             break;
1676         }
1677
1678         default:
1679             DFG_CRASH(m_graph, m_node, "Bad use kind");
1680             break;
1681         }
1682     }
1683
1684     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1685
1686     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1687
1688     void compileArithPow()
1689     {
1690         // FIXME: investigate llvm.powi to better understand its performance characteristics.
1691         // It might be better to have the inline loop in DFG too.
1692         if (m_node->child2().useKind() == Int32Use)
1693             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1694         else {
1695             LValue base = lowDouble(m_node->child1());
1696             LValue exponent = lowDouble(m_node->child2());
1697
1698             LBasicBlock integerExponentIsSmallBlock = FTL_NEW_BLOCK(m_out, ("ArithPow test integer exponent is small."));
1699             LBasicBlock integerExponentPowBlock = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, (int)double)."));
1700             LBasicBlock doubleExponentPowBlockEntry = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, double)."));
1701             LBasicBlock nanExceptionExponentIsInfinity = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check exponent is infinity."));
1702             LBasicBlock nanExceptionBaseIsOne = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check base is one."));
1703             LBasicBlock powBlock = FTL_NEW_BLOCK(m_out, ("ArithPow regular pow"));
1704             LBasicBlock nanExceptionResultIsNaN = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, result is NaN."));
1705             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithPow continuation"));
1706
1707             LValue integerExponent = m_out.fpToInt32(exponent);
1708             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1709             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1710             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1711
1712             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1713             LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1714             m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1715
1716             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1717             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1718             m_out.jump(continuation);
1719
1720             // If y is NaN, the result is NaN.
1721             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1722             LValue exponentIsNaN;
1723             if (m_state.forNode(m_node->child2()).m_type & SpecDoubleNaN)
1724                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1725             else
1726                 exponentIsNaN = m_out.booleanFalse;
1727             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1728
1729             // If abs(x) is 1 and y is +infinity, the result is NaN.
1730             // If abs(x) is 1 and y is -infinity, the result is NaN.
1731             m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1732             LValue absoluteExponent = m_out.doubleAbs(exponent);
1733             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1734             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1735
1736             m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1737             LValue absoluteBase = m_out.doubleAbs(base);
1738             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1739             m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1740
1741             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1742             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1743             m_out.jump(continuation);
1744
1745             m_out.appendTo(nanExceptionResultIsNaN, continuation);
1746             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1747             m_out.jump(continuation);
1748
1749             m_out.appendTo(continuation, lastNext);
1750             setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1751         }
1752     }
1753
1754     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1755
1756     void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
1757
1758     void compileArithFRound()
1759     {
1760         LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1761         setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1762     }
1763
1764     void compileArithNegate()
1765     {
1766         switch (m_node->child1().useKind()) {
1767         case Int32Use: {
1768             LValue value = lowInt32(m_node->child1());
1769
1770             LValue result;
1771             if (!shouldCheckOverflow(m_node->arithMode()))
1772                 result = m_out.neg(value);
1773             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1774                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1775                 // does the trick, though.
1776                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1777                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1778                 result = m_out.extractValue(overflowResult, 0);
1779             } else {
1780                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1781                 result = m_out.neg(value);
1782             }
1783
1784             setInt32(result);
1785             break;
1786         }
1787
1788         case Int52RepUse: {
1789             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)) {
1790                 Int52Kind kind;
1791                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1792                 LValue result = m_out.neg(value);
1793                 if (shouldCheckNegativeZero(m_node->arithMode()))
1794                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1795                 setInt52(result, kind);
1796                 break;
1797             }
1798
1799             LValue value = lowInt52(m_node->child1());
1800             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1801             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1802             LValue result = m_out.extractValue(overflowResult, 0);
1803             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1804             setInt52(result);
1805             break;
1806         }
1807
1808         case DoubleRepUse: {
1809             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1810             break;
1811         }
1812
1813         default:
1814             DFG_CRASH(m_graph, m_node, "Bad use kind");
1815             break;
1816         }
1817     }
1818
1819     void compileBitAnd()
1820     {
1821         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1822     }
1823
1824     void compileBitOr()
1825     {
1826         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1827     }
1828
1829     void compileBitXor()
1830     {
1831         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1832     }
1833
1834     void compileBitRShift()
1835     {
1836         setInt32(m_out.aShr(
1837             lowInt32(m_node->child1()),
1838             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1839     }
1840
1841     void compileBitLShift()
1842     {
1843         setInt32(m_out.shl(
1844             lowInt32(m_node->child1()),
1845             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1846     }
1847
1848     void compileBitURShift()
1849     {
1850         setInt32(m_out.lShr(
1851             lowInt32(m_node->child1()),
1852             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1853     }
1854
1855     void compileUInt32ToNumber()
1856     {
1857         LValue value = lowInt32(m_node->child1());
1858
1859         if (doesOverflow(m_node->arithMode())) {
1860             setDouble(m_out.unsignedToDouble(value));
1861             return;
1862         }
1863
1864         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
1865         setInt32(value);
1866     }
1867
1868     void compileCheckStructure()
1869     {
1870         LValue cell = lowCell(m_node->child1());
1871
1872         ExitKind exitKind;
1873         if (m_node->child1()->hasConstant())
1874             exitKind = BadConstantCache;
1875         else
1876             exitKind = BadCache;
1877
1878         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1879
1880         checkStructure(structureID, jsValueValue(cell), exitKind, m_node->structureSet());
1881     }
1882
1883     void compileCheckCell()
1884     {
1885         LValue cell = lowCell(m_node->child1());
1886
1887         speculate(
1888             BadCell, jsValueValue(cell), m_node->child1().node(),
1889             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
1890     }
1891
1892     void compileCheckBadCell()
1893     {
1894         terminate(BadCell);
1895     }
1896
1897     void compileCheckNotEmpty()
1898     {
1899         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
1900     }
1901
1902     void compileGetExecutable()
1903     {
1904         LValue cell = lowCell(m_node->child1());
1905         speculateFunction(m_node->child1(), cell);
1906         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
1907     }
1908
1909     void compileArrayifyToStructure()
1910     {
1911         LValue cell = lowCell(m_node->child1());
1912         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1913
1914         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1915         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1916
1917         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1918
1919         m_out.branch(
1920             m_out.notEqual(structureID, weakStructureID(m_node->structure())),
1921             rarely(unexpectedStructure), usually(continuation));
1922
1923         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1924
1925         if (property) {
1926             switch (m_node->arrayMode().type()) {
1927             case Array::Int32:
1928             case Array::Double:
1929             case Array::Contiguous:
1930                 speculate(
1931                     Uncountable, noValue(), 0,
1932                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1933                 break;
1934             default:
1935                 break;
1936             }
1937         }
1938
1939         switch (m_node->arrayMode().type()) {
1940         case Array::Int32:
1941             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1942             break;
1943         case Array::Double:
1944             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1945             break;
1946         case Array::Contiguous:
1947             if (m_node->arrayMode().conversion() == Array::RageConvert)
1948                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1949             else
1950                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1951             break;
1952         case Array::ArrayStorage:
1953         case Array::SlowPutArrayStorage:
1954             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1955             break;
1956         default:
1957             DFG_CRASH(m_graph, m_node, "Bad array type");
1958             break;
1959         }
1960
1961         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1962         speculate(
1963             BadIndexingType, jsValueValue(cell), 0,
1964             m_out.notEqual(structureID, weakStructureID(m_node->structure())));
1965         m_out.jump(continuation);
1966
1967         m_out.appendTo(continuation, lastNext);
1968     }
1969
1970     void compilePutStructure()
1971     {
1972         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1973
1974         Structure* oldStructure = m_node->transition()->previous;
1975         Structure* newStructure = m_node->transition()->next;
1976         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
1977         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
1978         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
1979
1980         LValue cell = lowCell(m_node->child1());
1981         m_out.store32(
1982             weakStructureID(newStructure),
1983             cell, m_heaps.JSCell_structureID);
1984     }
1985
1986     void compileGetById()
1987     {
1988         // Pretty much the only reason why we don't also support GetByIdFlush is because:
1989         // https://bugs.webkit.org/show_bug.cgi?id=125711
1990
1991         switch (m_node->child1().useKind()) {
1992         case CellUse: {
1993             setJSValue(getById(lowCell(m_node->child1())));
1994             return;
1995         }
1996
1997         case UntypedUse: {
1998             // This is pretty weird, since we duplicate the slow path both here and in the
1999             // code generated by the IC. We should investigate making this less bad.
2000             // https://bugs.webkit.org/show_bug.cgi?id=127830
2001             LValue value = lowJSValue(m_node->child1());
2002
2003             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
2004             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
2005             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
2006
2007             m_out.branch(isCell(value), unsure(cellCase), unsure(notCellCase));
2008
2009             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2010             ValueFromBlock cellResult = m_out.anchor(getById(value));
2011             m_out.jump(continuation);
2012
2013             m_out.appendTo(notCellCase, continuation);
2014             ValueFromBlock notCellResult = m_out.anchor(vmCall(
2015                 m_out.operation(operationGetById),
2016                 m_callFrame, getUndef(m_out.intPtr), value,
2017                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2018             m_out.jump(continuation);
2019
2020             m_out.appendTo(continuation, lastNext);
2021             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2022             return;
2023         }
2024
2025         default:
2026             DFG_CRASH(m_graph, m_node, "Bad use kind");
2027             return;
2028         }
2029     }
2030
2031     void compilePutById()
2032     {
2033         // See above; CellUse is easier so we do only that for now.
2034         ASSERT(m_node->child1().useKind() == CellUse);
2035
2036         LValue base = lowCell(m_node->child1());
2037         LValue value = lowJSValue(m_node->child2());
2038         StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
2039
2040         // Arguments: id, bytes, target, numArgs, args...
2041         unsigned stackmapID = m_stackmapIDs++;
2042
2043         if (verboseCompilationEnabled())
2044             dataLog("    Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
2045
2046         LValue call = m_out.call(
2047             m_out.patchpointVoidIntrinsic(),
2048             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
2049             constNull(m_out.ref8), m_out.constInt32(2), base, value);
2050         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
2051
2052         m_ftlState.putByIds.append(PutByIdDescriptor(
2053             stackmapID, m_node->origin.semantic, uid,
2054             m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
2055             m_node->op() == PutByIdDirect ? Direct : NotDirect));
2056     }
2057
2058     void compileGetButterfly()
2059     {
2060         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2061     }
2062
2063     void compileConstantStoragePointer()
2064     {
2065         setStorage(m_out.constIntPtr(m_node->storagePointer()));
2066     }
2067
2068     void compileGetIndexedPropertyStorage()
2069     {
2070         LValue cell = lowCell(m_node->child1());
2071
2072         if (m_node->arrayMode().type() == Array::String) {
2073             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
2074             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
2075
2076             ValueFromBlock fastResult = m_out.anchor(
2077                 m_out.loadPtr(cell, m_heaps.JSString_value));
2078
2079             m_out.branch(
2080                 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
2081
2082             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2083
2084             ValueFromBlock slowResult = m_out.anchor(
2085                 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
2086
2087             m_out.jump(continuation);
2088
2089             m_out.appendTo(continuation, lastNext);
2090
2091             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2092             return;
2093         }
2094
2095         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2096     }
2097
2098     void compileCheckArray()
2099     {
2100         Edge edge = m_node->child1();
2101         LValue cell = lowCell(edge);
2102
2103         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge)))
2104             return;
2105
2106         speculate(
2107             BadIndexingType, jsValueValue(cell), 0,
2108             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
2109     }
2110
2111     void compileGetTypedArrayByteOffset()
2112     {
2113         LValue basePtr = lowCell(m_node->child1());
2114
2115         LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
2116         LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
2117         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
2118
2119         LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2120         m_out.branch(
2121             m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2122             unsure(simpleCase), unsure(wastefulCase));
2123
2124         // begin simple case
2125         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2126
2127         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2128
2129         m_out.jump(continuation);
2130
2131         // begin wasteful case
2132         m_out.appendTo(wastefulCase, continuation);
2133
2134         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2135         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2136         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2137         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2138
2139         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2140
2141         m_out.jump(continuation);
2142         m_out.appendTo(continuation, lastNext);
2143
2144         // output
2145         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2146     }
2147
2148     void compileGetArrayLength()
2149     {
2150         switch (m_node->arrayMode().type()) {
2151         case Array::Int32:
2152         case Array::Double:
2153         case Array::Contiguous: {
2154             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2155             return;
2156         }
2157
2158         case Array::String: {
2159             LValue string = lowCell(m_node->child1());
2160             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2161             return;
2162         }
2163
2164         case Array::DirectArguments: {
2165             LValue arguments = lowCell(m_node->child1());
2166             speculate(
2167                 ExoticObjectMode, noValue(), nullptr,
2168                 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2169             setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2170             return;
2171         }
2172
2173         case Array::ScopedArguments: {
2174             LValue arguments = lowCell(m_node->child1());
2175             speculate(
2176                 ExoticObjectMode, noValue(), nullptr,
2177                 m_out.notZero8(m_out.load8(arguments, m_heaps.ScopedArguments_overrodeThings)));
2178             setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2179             return;
2180         }
2181
2182         default:
2183             if (isTypedView(m_node->arrayMode().typedArrayType())) {
2184                 setInt32(
2185                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2186                 return;
2187             }
2188
2189             DFG_CRASH(m_graph, m_node, "Bad array type");
2190             return;
2191         }
2192     }
2193
2194     void compileCheckInBounds()
2195     {
2196         speculate(
2197             OutOfBounds, noValue(), 0,
2198             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2199     }
2200
2201     void compileGetByVal()
2202     {
2203         switch (m_node->arrayMode().type()) {
2204         case Array::Int32:
2205         case Array::Contiguous: {
2206             LValue index = lowInt32(m_node->child2());
2207             LValue storage = lowStorage(m_node->child3());
2208
2209             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2210                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2211
2212             if (m_node->arrayMode().isInBounds()) {
2213                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2214                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
2215                 setJSValue(result);
2216                 return;
2217             }
2218
2219             LValue base = lowCell(m_node->child1());
2220
2221             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2222             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2223             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2224
2225             m_out.branch(
2226                 m_out.aboveOrEqual(
2227                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2228                 rarely(slowCase), usually(fastCase));
2229
2230             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2231
2232             ValueFromBlock fastResult = m_out.anchor(
2233                 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2234             m_out.branch(
2235                 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2236
2237             m_out.appendTo(slowCase, continuation);
2238             ValueFromBlock slowResult = m_out.anchor(
2239                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2240             m_out.jump(continuation);
2241
2242             m_out.appendTo(continuation, lastNext);
2243             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2244             return;
2245         }
2246
2247         case Array::Double: {
2248             LValue index = lowInt32(m_node->child2());
2249             LValue storage = lowStorage(m_node->child3());
2250
2251             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2252
2253             if (m_node->arrayMode().isInBounds()) {
2254                 LValue result = m_out.loadDouble(
2255                     baseIndex(heap, storage, index, m_node->child2()));
2256
2257                 if (!m_node->arrayMode().isSaneChain()) {
2258                     speculate(
2259                         LoadFromHole, noValue(), 0,
2260                         m_out.doubleNotEqualOrUnordered(result, result));
2261                 }
2262                 setDouble(result);
2263                 break;
2264             }
2265
2266             LValue base = lowCell(m_node->child1());
2267
2268             LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2269             LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2270             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2271             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2272
2273             m_out.branch(
2274                 m_out.aboveOrEqual(
2275                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2276                 rarely(slowCase), usually(inBounds));
2277
2278             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2279             LValue doubleValue = m_out.loadDouble(
2280                 baseIndex(heap, storage, index, m_node->child2()));
2281             m_out.branch(
2282                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2283                 rarely(slowCase), usually(boxPath));
2284
2285             m_out.appendTo(boxPath, slowCase);
2286             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2287             m_out.jump(continuation);
2288
2289             m_out.appendTo(slowCase, continuation);
2290             ValueFromBlock slowResult = m_out.anchor(
2291                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2292             m_out.jump(continuation);
2293
2294             m_out.appendTo(continuation, lastNext);
2295             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2296             return;
2297         }
2298
2299         case Array::DirectArguments: {
2300             LValue base = lowCell(m_node->child1());
2301             LValue index = lowInt32(m_node->child2());
2302
2303             speculate(
2304                 ExoticObjectMode, noValue(), nullptr,
2305                 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2306             speculate(
2307                 ExoticObjectMode, noValue(), nullptr,
2308                 m_out.aboveOrEqual(
2309                     index,
2310                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2311
2312             TypedPointer address = m_out.baseIndex(
2313                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2314             setJSValue(m_out.load64(address));
2315             return;
2316         }
2317
2318         case Array::ScopedArguments: {
2319             LValue base = lowCell(m_node->child1());
2320             LValue index = lowInt32(m_node->child2());
2321
2322             speculate(
2323                 ExoticObjectMode, noValue(), nullptr,
2324                 m_out.aboveOrEqual(
2325                     index,
2326                     m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2327
2328             LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2329             LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2330
2331             LBasicBlock namedCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments named case"));
2332             LBasicBlock overflowCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments overflow case"));
2333             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments continuation"));
2334
2335             m_out.branch(
2336                 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2337
2338             LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2339
2340             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2341             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2342
2343             TypedPointer address = m_out.baseIndex(
2344                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2345             LValue scopeOffset = m_out.load32(address);
2346
2347             speculate(
2348                 ExoticObjectMode, noValue(), nullptr,
2349                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2350
2351             address = m_out.baseIndex(
2352                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2353             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2354             m_out.jump(continuation);
2355
2356             m_out.appendTo(overflowCase, continuation);
2357
2358             address = m_out.baseIndex(
2359                 m_heaps.ScopedArguments_overflowStorage, base,
2360                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2361             LValue overflowValue = m_out.load64(address);
2362             speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2363             ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2364             m_out.jump(continuation);
2365
2366             m_out.appendTo(continuation, lastNext);
2367             setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2368             return;
2369         }
2370
2371         case Array::Generic: {
2372             setJSValue(vmCall(
2373                 m_out.operation(operationGetByVal), m_callFrame,
2374                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2375             return;
2376         }
2377
2378         case Array::String: {
2379             compileStringCharAt();
2380             return;
2381         }
2382
2383         default: {
2384             LValue index = lowInt32(m_node->child2());
2385             LValue storage = lowStorage(m_node->child3());
2386
2387             TypedArrayType type = m_node->arrayMode().typedArrayType();
2388
2389             if (isTypedView(type)) {
2390                 TypedPointer pointer = TypedPointer(
2391                     m_heaps.typedArrayProperties,
2392                     m_out.add(
2393                         storage,
2394                         m_out.shl(
2395                             m_out.zeroExtPtr(index),
2396                             m_out.constIntPtr(logElementSize(type)))));
2397
2398                 if (isInt(type)) {
2399                     LValue result;
2400                     switch (elementSize(type)) {
2401                     case 1:
2402                         result = m_out.load8(pointer);
2403                         break;
2404                     case 2:
2405                         result = m_out.load16(pointer);
2406                         break;
2407                     case 4:
2408                         result = m_out.load32(pointer);
2409                         break;
2410                     default:
2411                         DFG_CRASH(m_graph, m_node, "Bad element size");
2412                     }
2413
2414                     if (elementSize(type) < 4) {
2415                         if (isSigned(type))
2416                             result = m_out.signExt(result, m_out.int32);
2417                         else
2418                             result = m_out.zeroExt(result, m_out.int32);
2419                         setInt32(result);
2420                         return;
2421                     }
2422
2423                     if (isSigned(type)) {
2424                         setInt32(result);
2425                         return;
2426                     }
2427
2428                     if (m_node->shouldSpeculateInt32()) {
2429                         speculate(
2430                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2431                         setInt32(result);
2432                         return;
2433                     }
2434
2435                     if (m_node->shouldSpeculateMachineInt()) {
2436                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2437                         return;
2438                     }
2439
2440                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2441                     return;
2442                 }
2443
2444                 ASSERT(isFloat(type));
2445
2446                 LValue result;
2447                 switch (type) {
2448                 case TypeFloat32:
2449                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2450                     break;
2451                 case TypeFloat64:
2452                     result = m_out.loadDouble(pointer);
2453                     break;
2454                 default:
2455                     DFG_CRASH(m_graph, m_node, "Bad typed array type");
2456                 }
2457
2458                 setDouble(result);
2459                 return;
2460             }
2461
2462             DFG_CRASH(m_graph, m_node, "Bad array type");
2463             return;
2464         } }
2465     }
2466
2467     void compileGetMyArgumentByVal()
2468     {
2469         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2470
2471         LValue index = lowInt32(m_node->child2());
2472
2473         LValue limit;
2474         if (inlineCallFrame && !inlineCallFrame->isVarargs())
2475             limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2476         else {
2477             VirtualRegister argumentCountRegister;
2478             if (!inlineCallFrame)
2479                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2480             else
2481                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2482             limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2483         }
2484
2485         speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2486
2487         TypedPointer base;
2488         if (inlineCallFrame) {
2489             if (inlineCallFrame->arguments.size() <= 1) {
2490                 // We should have already exited due to the bounds check, above. Just tell the
2491                 // compiler that anything dominated by this instruction is not reachable, so
2492                 // that we don't waste time generating such code. This will also plant some
2493                 // kind of crashing instruction so that if by some fluke the bounds check didn't
2494                 // work, we'll crash in an easy-to-see way.
2495                 didAlreadyTerminate();
2496                 return;
2497             }
2498             base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2499         } else
2500             base = addressFor(virtualRegisterForArgument(1));
2501
2502         LValue pointer = m_out.baseIndex(
2503             base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2504         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2505     }
2506
2507     void compilePutByVal()
2508     {
2509         Edge child1 = m_graph.varArgChild(m_node, 0);
2510         Edge child2 = m_graph.varArgChild(m_node, 1);
2511         Edge child3 = m_graph.varArgChild(m_node, 2);
2512         Edge child4 = m_graph.varArgChild(m_node, 3);
2513         Edge child5 = m_graph.varArgChild(m_node, 4);
2514
2515         switch (m_node->arrayMode().type()) {
2516         case Array::Generic: {
2517             V_JITOperation_EJJJ operation;
2518             if (m_node->op() == PutByValDirect) {
2519                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2520                     operation = operationPutByValDirectStrict;
2521                 else
2522                     operation = operationPutByValDirectNonStrict;
2523             } else {
2524                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2525                     operation = operationPutByValStrict;
2526                 else
2527                     operation = operationPutByValNonStrict;
2528             }
2529
2530             vmCall(
2531                 m_out.operation(operation), m_callFrame,
2532                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2533             return;
2534         }
2535
2536         default:
2537             break;
2538         }
2539
2540         LValue base = lowCell(child1);
2541         LValue index = lowInt32(child2);
2542         LValue storage = lowStorage(child4);
2543
2544         switch (m_node->arrayMode().type()) {
2545         case Array::Int32:
2546         case Array::Double:
2547         case Array::Contiguous: {
2548             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2549             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2550
2551             switch (m_node->arrayMode().type()) {
2552             case Array::Int32:
2553             case Array::Contiguous: {
2554                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2555
2556                 if (m_node->arrayMode().type() == Array::Int32)
2557                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2558
2559                 TypedPointer elementPointer = m_out.baseIndex(
2560                     m_node->arrayMode().type() == Array::Int32 ?
2561                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2562                     storage, m_out.zeroExtPtr(index), m_state.forNode(child2).m_value);
2563
2564                 if (m_node->op() == PutByValAlias) {
2565                     m_out.store64(value, elementPointer);
2566                     break;
2567                 }
2568
2569                 contiguousPutByValOutOfBounds(
2570                     codeBlock()->isStrictMode()
2571                     ? operationPutByValBeyondArrayBoundsStrict
2572                     : operationPutByValBeyondArrayBoundsNonStrict,
2573                     base, storage, index, value, continuation);
2574
2575                 m_out.store64(value, elementPointer);
2576                 break;
2577             }
2578
2579             case Array::Double: {
2580                 LValue value = lowDouble(child3);
2581
2582                 FTL_TYPE_CHECK(
2583                     doubleValue(value), child3, SpecDoubleReal,
2584                     m_out.doubleNotEqualOrUnordered(value, value));
2585
2586                 TypedPointer elementPointer = m_out.baseIndex(
2587                     m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2588                     m_state.forNode(child2).m_value);
2589
2590                 if (m_node->op() == PutByValAlias) {
2591                     m_out.storeDouble(value, elementPointer);
2592                     break;
2593                 }
2594
2595                 contiguousPutByValOutOfBounds(
2596                     codeBlock()->isStrictMode()
2597                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2598                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2599                     base, storage, index, value, continuation);
2600
2601                 m_out.storeDouble(value, elementPointer);
2602                 break;
2603             }
2604
2605             default:
2606                 DFG_CRASH(m_graph, m_node, "Bad array type");
2607             }
2608
2609             m_out.jump(continuation);
2610             m_out.appendTo(continuation, outerLastNext);
2611             return;
2612         }
2613
2614         default:
2615             TypedArrayType type = m_node->arrayMode().typedArrayType();
2616
2617             if (isTypedView(type)) {
2618                 TypedPointer pointer = TypedPointer(
2619                     m_heaps.typedArrayProperties,
2620                     m_out.add(
2621                         storage,
2622                         m_out.shl(
2623                             m_out.zeroExt(index, m_out.intPtr),
2624                             m_out.constIntPtr(logElementSize(type)))));
2625
2626                 LType refType;
2627                 LValue valueToStore;
2628
2629                 if (isInt(type)) {
2630                     LValue intValue;
2631                     switch (child3.useKind()) {
2632                     case Int52RepUse:
2633                     case Int32Use: {
2634                         if (child3.useKind() == Int32Use)
2635                             intValue = lowInt32(child3);
2636                         else
2637                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2638
2639                         if (isClamped(type)) {
2640                             ASSERT(elementSize(type) == 1);
2641
2642                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2643                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2644
2645                             Vector<ValueFromBlock, 2> intValues;
2646                             intValues.append(m_out.anchor(m_out.int32Zero));
2647                             m_out.branch(
2648                                 m_out.lessThan(intValue, m_out.int32Zero),
2649                                 unsure(continuation), unsure(atLeastZero));
2650
2651                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2652
2653                             intValues.append(m_out.anchor(m_out.select(
2654                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
2655                                 m_out.constInt32(255),
2656                                 intValue)));
2657                             m_out.jump(continuation);
2658
2659                             m_out.appendTo(continuation, lastNext);
2660                             intValue = m_out.phi(m_out.int32, intValues);
2661                         }
2662                         break;
2663                     }
2664
2665                     case DoubleRepUse: {
2666                         LValue doubleValue = lowDouble(child3);
2667
2668                         if (isClamped(type)) {
2669                             ASSERT(elementSize(type) == 1);
2670
2671                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2672                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2673                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2674
2675                             Vector<ValueFromBlock, 3> intValues;
2676                             intValues.append(m_out.anchor(m_out.int32Zero));
2677                             m_out.branch(
2678                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2679                                 unsure(continuation), unsure(atLeastZero));
2680
2681                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2682                             intValues.append(m_out.anchor(m_out.constInt32(255)));
2683                             m_out.branch(
2684                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2685                                 unsure(continuation), unsure(withinRange));
2686
2687                             m_out.appendTo(withinRange, continuation);
2688                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2689                             m_out.jump(continuation);
2690
2691                             m_out.appendTo(continuation, lastNext);
2692                             intValue = m_out.phi(m_out.int32, intValues);
2693                         } else
2694                             intValue = doubleToInt32(doubleValue);
2695                         break;
2696                     }
2697
2698                     default:
2699                         DFG_CRASH(m_graph, m_node, "Bad use kind");
2700                     }
2701
2702                     switch (elementSize(type)) {
2703                     case 1:
2704                         valueToStore = m_out.intCast(intValue, m_out.int8);
2705                         refType = m_out.ref8;
2706                         break;
2707                     case 2:
2708                         valueToStore = m_out.intCast(intValue, m_out.int16);
2709                         refType = m_out.ref16;
2710                         break;
2711                     case 4:
2712                         valueToStore = intValue;
2713                         refType = m_out.ref32;
2714                         break;
2715                     default:
2716                         DFG_CRASH(m_graph, m_node, "Bad element size");
2717                     }
2718                 } else /* !isInt(type) */ {
2719                     LValue value = lowDouble(child3);
2720                     switch (type) {
2721                     case TypeFloat32:
2722                         valueToStore = m_out.fpCast(value, m_out.floatType);
2723                         refType = m_out.refFloat;
2724                         break;
2725                     case TypeFloat64:
2726                         valueToStore = value;
2727                         refType = m_out.refDouble;
2728                         break;
2729                     default:
2730                         DFG_CRASH(m_graph, m_node, "Bad typed array type");
2731                     }
2732                 }
2733
2734                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2735                     m_out.store(valueToStore, pointer, refType);
2736                 else {
2737                     LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2738                     LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2739
2740                     m_out.branch(
2741                         m_out.aboveOrEqual(index, lowInt32(child5)),
2742                         unsure(continuation), unsure(isInBounds));
2743
2744                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2745                     m_out.store(valueToStore, pointer, refType);
2746                     m_out.jump(continuation);
2747
2748                     m_out.appendTo(continuation, lastNext);
2749                 }
2750
2751                 return;
2752             }
2753
2754             DFG_CRASH(m_graph, m_node, "Bad array type");
2755             break;
2756         }
2757     }
2758
2759     void compileArrayPush()
2760     {
2761         LValue base = lowCell(m_node->child1());
2762         LValue storage = lowStorage(m_node->child3());
2763
2764         switch (m_node->arrayMode().type()) {
2765         case Array::Int32:
2766         case Array::Contiguous:
2767         case Array::Double: {
2768             LValue value;
2769             LType refType;
2770
2771             if (m_node->arrayMode().type() != Array::Double) {
2772                 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2773                 if (m_node->arrayMode().type() == Array::Int32) {
2774                     FTL_TYPE_CHECK(
2775                         jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2776                 }
2777                 refType = m_out.ref64;
2778             } else {
2779                 value = lowDouble(m_node->child2());
2780                 FTL_TYPE_CHECK(
2781                     doubleValue(value), m_node->child2(), SpecDoubleReal,
2782                     m_out.doubleNotEqualOrUnordered(value, value));
2783                 refType = m_out.refDouble;
2784             }
2785
2786             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2787
2788             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2789
2790             LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2791             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2792             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2793
2794             m_out.branch(
2795                 m_out.aboveOrEqual(
2796                     prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2797                 rarely(slowPath), usually(fastPath));
2798
2799             LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2800             m_out.store(
2801                 value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), refType);
2802             LValue newLength = m_out.add(prevLength, m_out.int32One);
2803             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2804
2805             ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2806             m_out.jump(continuation);
2807
2808             m_out.appendTo(slowPath, continuation);
2809             LValue operation;
2810             if (m_node->arrayMode().type() != Array::Double)
2811                 operation = m_out.operation(operationArrayPush);
2812             else
2813                 operation = m_out.operation(operationArrayPushDouble);
2814             ValueFromBlock slowResult = m_out.anchor(
2815                 vmCall(operation, m_callFrame, value, base));
2816             m_out.jump(continuation);
2817
2818             m_out.appendTo(continuation, lastNext);
2819             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2820             return;
2821         }
2822
2823         default:
2824             DFG_CRASH(m_graph, m_node, "Bad array type");
2825             return;
2826         }
2827     }
2828
2829     void compileArrayPop()
2830     {
2831         LValue base = lowCell(m_node->child1());
2832         LValue storage = lowStorage(m_node->child2());
2833
2834         switch (m_node->arrayMode().type()) {
2835         case Array::Int32:
2836         case Array::Double:
2837         case Array::Contiguous: {
2838             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2839
2840             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
2841             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
2842             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
2843
2844             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2845
2846             Vector<ValueFromBlock, 3> results;
2847             results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
2848             m_out.branch(
2849                 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
2850
2851             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2852             LValue newLength = m_out.sub(prevLength, m_out.int32One);
2853             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2854             TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength));
2855             if (m_node->arrayMode().type() != Array::Double) {
2856                 LValue result = m_out.load64(pointer);
2857                 m_out.store64(m_out.int64Zero, pointer);
2858                 results.append(m_out.anchor(result));
2859                 m_out.branch(
2860                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2861             } else {
2862                 LValue result = m_out.loadDouble(pointer);
2863                 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
2864                 results.append(m_out.anchor(boxDouble(result)));
2865                 m_out.branch(
2866                     m_out.doubleEqual(result, result),
2867                     usually(continuation), rarely(slowCase));
2868             }
2869
2870             m_out.appendTo(slowCase, continuation);
2871             results.append(m_out.anchor(vmCall(
2872                 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
2873             m_out.jump(continuation);
2874
2875             m_out.appendTo(continuation, lastNext);
2876             setJSValue(m_out.phi(m_out.int64, results));
2877             return;
2878         }
2879
2880         default:
2881             DFG_CRASH(m_graph, m_node, "Bad array type");
2882             return;
2883         }
2884     }
2885
2886     void compileCreateActivation()
2887     {
2888         LValue scope = lowCell(m_node->child1());
2889         SymbolTable* table = m_graph.symbolTableFor(m_node->origin.semantic);
2890         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
2891
2892         if (table->singletonScope()->isStillValid()) {
2893             LValue callResult = vmCall(
2894                 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
2895                 scope, weakPointer(table));
2896             setJSValue(callResult);
2897             return;
2898         }
2899
2900         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateActivation slow path"));
2901         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateActivation continuation"));
2902
2903         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2904
2905         LValue fastObject = allocateObject<JSLexicalEnvironment>(
2906             JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
2907
2908         // We don't need memory barriers since we just fast-created the activation, so the
2909         // activation must be young.
2910         m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
2911         m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
2912
2913         for (unsigned i = 0; i < table->scopeSize(); ++i) {
2914             m_out.store64(
2915                 m_out.constInt64(JSValue::encode(jsUndefined())),
2916                 fastObject, m_heaps.JSEnvironmentRecord_variables[i]);
2917         }
2918
2919         ValueFromBlock fastResult = m_out.anchor(fastObject);
2920         m_out.jump(continuation);
2921
2922         m_out.appendTo(slowPath, continuation);
2923         LValue callResult = vmCall(
2924             m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
2925             scope, weakPointer(table));
2926         ValueFromBlock slowResult = m_out.anchor(callResult);
2927         m_out.jump(continuation);
2928
2929         m_out.appendTo(continuation, lastNext);
2930         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2931     }
2932
2933     void compileNewFunction()
2934     {
2935         LValue scope = lowCell(m_node->child1());
2936         FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>();
2937         if (executable->singletonFunction()->isStillValid()) {
2938             LValue callResult = vmCall(
2939                 m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable));
2940             setJSValue(callResult);
2941             return;
2942         }
2943
2944         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure();
2945
2946         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewFunction slow path"));
2947         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewFunction continuation"));
2948
2949         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2950
2951         LValue fastObject = allocateObject<JSFunction>(
2952             structure, m_out.intPtrZero, slowPath);
2953
2954         // We don't need memory barriers since we just fast-created the function, so it
2955         // must be young.
2956         m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope);
2957         m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable);
2958         m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData);
2959
2960         ValueFromBlock fastResult = m_out.anchor(fastObject);
2961         m_out.jump(continuation);
2962
2963         m_out.appendTo(slowPath, continuation);
2964         LValue callResult = vmCall(
2965             m_out.operation(operationNewFunctionWithInvalidatedReallocationWatchpoint),
2966             m_callFrame, scope, weakPointer(executable));
2967         ValueFromBlock slowResult = m_out.anchor(callResult);
2968         m_out.jump(continuation);
2969
2970         m_out.appendTo(continuation, lastNext);
2971         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2972     }
2973
2974     void compileCreateDirectArguments()
2975     {
2976         // FIXME: A more effective way of dealing with the argument count and callee is to have
2977         // them be explicit arguments to this node.
2978         // https://bugs.webkit.org/show_bug.cgi?id=142207
2979
2980         Structure* structure =
2981             m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure();
2982
2983         unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1;
2984
2985         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments slow path"));
2986         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments continuation"));
2987
2988         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2989
2990         ArgumentsLength length = getArgumentsLength();
2991
2992         LValue fastObject;
2993         if (length.isKnown) {
2994             fastObject = allocateObject<DirectArguments>(
2995                 DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure,
2996                 m_out.intPtrZero, slowPath);
2997         } else {
2998             LValue size = m_out.add(
2999                 m_out.shl(length.value, m_out.constInt32(3)),
3000                 m_out.constInt32(DirectArguments::storageOffset()));
3001
3002             size = m_out.select(
3003                 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3004                 size, m_out.constInt32(DirectArguments::allocationSize(minCapacity)));
3005
3006             fastObject = allocateVariableSizedObject<DirectArguments>(
3007                 size, structure, m_out.intPtrZero, slowPath);
3008         }
3009
3010         m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length);
3011         m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity);
3012         m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_overrides);
3013
3014         ValueFromBlock fastResult = m_out.anchor(fastObject);
3015         m_out.jump(continuation);
3016
3017         m_out.appendTo(slowPath, continuation);
3018         LValue callResult = vmCall(
3019             m_out.operation(operationCreateDirectArguments), m_callFrame, weakPointer(structure),
3020             length.value, m_out.constInt32(minCapacity));
3021         ValueFromBlock slowResult = m_out.anchor(callResult);
3022         m_out.jump(continuation);
3023
3024         m_out.appendTo(continuation, lastNext);
3025         LValue result = m_out.phi(m_out.intPtr, fastResult, slowResult);
3026
3027         m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee);
3028
3029         if (length.isKnown) {
3030             VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic);
3031             for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) {
3032                 m_out.store64(
3033                     m_out.load64(addressFor(start + i)),
3034                     result, m_heaps.DirectArguments_storage[i]);
3035             }
3036         } else {
3037             LValue stackBase = getArgumentsStart();
3038
3039             LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop body"));
3040             LBasicBlock end = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop end"));
3041
3042             ValueFromBlock originalLength;
3043             if (minCapacity) {
3044                 LValue capacity = m_out.select(
3045                     m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3046                     length.value,
3047                     m_out.constInt32(minCapacity));
3048                 originalLength = m_out.anchor(m_out.zeroExtPtr(capacity));
3049                 m_out.jump(loop);
3050             } else {
3051                 originalLength = m_out.anchor(m_out.zeroExtPtr(length.value));
3052                 m_out.branch(m_out.isNull(originalLength.value()), unsure(end), unsure(loop));
3053             }
3054
3055             lastNext = m_out.appendTo(loop, end);
3056             LValue previousIndex