Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp

   1 /*
   2  * Copyright (C) 2013-2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToB3.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "AirGenerationContext.h"
  32 #include "AllowMacroScratchRegisterUsage.h"
  33 #include "B3StackmapGenerationParams.h"
  34 #include "CallFrameShuffler.h"
  35 #include "CodeBlockWithJITType.h"
  36 #include "DFGAbstractInterpreterInlines.h"
  37 #include "DFGDominators.h"
  38 #include "DFGInPlaceAbstractState.h"
  39 #include "DFGOSRAvailabilityAnalysisPhase.h"
  40 #include "DFGOSRExitFuzz.h"
  41 #include "DirectArguments.h"
  42 #include "FTLAbstractHeapRepository.h"
  43 #include "FTLAvailableRecovery.h"
  44 #include "FTLExceptionTarget.h"
  45 #include "FTLForOSREntryJITCode.h"
  46 #include "FTLFormattedValue.h"
  47 #include "FTLLazySlowPathCall.h"
  48 #include "FTLLoweredNodeValue.h"
  49 #include "FTLOperations.h"
  50 #include "FTLOutput.h"
  51 #include "FTLPatchpointExceptionHandle.h"
  52 #include "FTLThunks.h"
  53 #include "FTLWeightedTarget.h"
  54 #include "JITAddGenerator.h"
  55 #include "JITBitAndGenerator.h"
  56 #include "JITBitOrGenerator.h"
  57 #include "JITBitXorGenerator.h"
  58 #include "JITDivGenerator.h"
  59 #include "JITInlineCacheGenerator.h"
  60 #include "JITLeftShiftGenerator.h"
  61 #include "JITMulGenerator.h"
  62 #include "JITRightShiftGenerator.h"
  63 #include "JITSubGenerator.h"
  64 #include "JSCInlines.h"
  65 #include "JSGeneratorFunction.h"
  66 #include "JSLexicalEnvironment.h"
  67 #include "OperandsInlines.h"
  68 #include "ScopedArguments.h"
  69 #include "ScopedArgumentsTable.h"
  70 #include "ScratchRegisterAllocator.h"
  71 #include "SetupVarargsFrame.h"
  72 #include "ShadowChicken.h"
  73 #include "StructureStubInfo.h"
  74 #include "VirtualRegister.h"
  75 #include "Watchdog.h"
  76 #include <atomic>
  77 #if !OS(WINDOWS)
  78 #include <dlfcn.h>
  79 #endif
  80 #include <unordered_set>
  81 #include <wtf/Box.h>
  82 #include <wtf/ProcessID.h>
  83
  84 namespace JSC { namespace FTL {
  85
  86 using namespace B3;
  87 using namespace DFG;
  88
  89 namespace {
  90
  91 std::atomic<int> compileCounter;
  92
  93 #if ASSERT_DISABLED
  94 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
  95 {
  96     CRASH();
  97 }
  98 #else
  99 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
 100     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
 101 {
 102     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
 103     if (nodeIndex != UINT_MAX)
 104         dataLog(", node @", nodeIndex);
 105     dataLog(".\n");
 106     CRASH();
 107 }
 108 #endif
 109
 110 // Using this instead of typeCheck() helps to reduce the load on B3, by creating
 111 // significantly less dead code.
 112 #define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \
 113         FormattedValue _ftc_lowValue = (lowValue);                      \
 114         Edge _ftc_highValue = (highValue);                              \
 115         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
 116         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
 117             break;                                                      \
 118         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \
 119     } while (false)
 120
 121 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \
 122     FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition)
 123
 124 class LowerDFGToB3 {
 125     WTF_MAKE_NONCOPYABLE(LowerDFGToB3);
 126 public:
 127     LowerDFGToB3(State& state)
 128         : m_graph(state.graph)
 129         , m_ftlState(state)
 130         , m_out(state)
 131         , m_proc(*state.proc)
 132         , m_state(state.graph)
 133         , m_interpreter(state.graph, m_state)
 134     {
 135     }
 136
 137     void lower()
 138     {
 139         State* state = &m_ftlState;
 140
 141         CString name;
 142         if (verboseCompilationEnabled()) {
 143             name = toCString(
 144                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
 145                 "_", codeBlock()->hash());
 146         } else
 147             name = "jsBody";
 148
 149         m_graph.ensureDominators();
 150
 151         if (verboseCompilationEnabled())
 152             dataLog("Function ready, beginning lowering.\n");
 153
 154         m_out.initialize(m_heaps);
 155
 156         // We use prologue frequency for all of the initialization code.
 157         m_out.setFrequency(1);
 158
 159         m_prologue = m_out.newBlock();
 160         LBasicBlock stackOverflow = m_out.newBlock();
 161         m_handleExceptions = m_out.newBlock();
 162
 163         LBasicBlock checkArguments = m_out.newBlock();
 164
 165         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 166             m_highBlock = m_graph.block(blockIndex);
 167             if (!m_highBlock)
 168                 continue;
 169             m_out.setFrequency(m_highBlock->executionCount);
 170             m_blocks.add(m_highBlock, m_out.newBlock());
 171         }
 172
 173         // Back to prologue frequency for any bocks that get sneakily created in the initialization code.
 174         m_out.setFrequency(1);
 175
 176         m_out.appendTo(m_prologue, stackOverflow);
 177         m_out.initializeConstants(m_proc, m_prologue);
 178         createPhiVariables();
 179
 180         size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;
 181         B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);
 182         m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));
 183         state->capturedValue = capturedBase->slot();
 184
 185         auto preOrder = m_graph.blocksInPreOrder();
 186
 187         // We should not create any alloca's after this point, since they will cease to
 188         // be mem2reg candidates.
 189
 190         m_callFrame = m_out.framePointer();
 191         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 192         m_tagMask = m_out.constInt64(TagMask);
 193
 194         // Make sure that B3 knows that we really care about the mask registers. This forces the
 195         // constants to be materialized in registers.
 196         m_proc.addFastConstant(m_tagTypeNumber->key());
 197         m_proc.addFastConstant(m_tagMask->key());
 198
 199         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 200
 201         m_out.branch(
 202             didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
 203
 204         m_out.appendTo(stackOverflow, m_handleExceptions);
 205         m_out.call(m_out.voidType, m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 206         m_out.patchpoint(Void)->setGenerator(
 207             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
 208                 // We are terminal, so we can clobber everything. That's why we don't claim to
 209                 // clobber scratch.
 210                 AllowMacroScratchRegisterUsage allowScratch(jit);
 211
 212                 jit.copyCalleeSavesToVMCalleeSavesBuffer();
 213                 jit.move(CCallHelpers::TrustedImmPtr(jit.vm()), GPRInfo::argumentGPR0);
 214                 jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
 215                 CCallHelpers::Call call = jit.call();
 216                 jit.jumpToExceptionHandler();
 217
 218                 jit.addLinkTask(
 219                     [=] (LinkBuffer& linkBuffer) {
 220                         linkBuffer.link(call, FunctionPtr(lookupExceptionHandlerFromCallerFrame));
 221                     });
 222             });
 223         m_out.unreachable();
 224
 225         m_out.appendTo(m_handleExceptions, checkArguments);
 226         Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;
 227         m_out.patchpoint(Void)->setGenerator(
 228             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
 229                 CCallHelpers::Jump jump = jit.jump();
 230                 jit.addLinkTask(
 231                     [=] (LinkBuffer& linkBuffer) {
 232                         linkBuffer.link(jump, linkBuffer.locationOf(*exceptionHandler));
 233                     });
 234             });
 235         m_out.unreachable();
 236
 237         m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
 238         availabilityMap().clear();
 239         availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
 240         for (unsigned i = codeBlock()->numParameters(); i--;) {
 241             availabilityMap().m_locals.argument(i) =
 242                 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
 243         }
 244         m_node = nullptr;
 245         m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
 246         for (unsigned i = codeBlock()->numParameters(); i--;) {
 247             Node* node = m_graph.m_arguments[i];
 248             VirtualRegister operand = virtualRegisterForArgument(i);
 249
 250             LValue jsValue = m_out.load64(addressFor(operand));
 251
 252             if (node) {
 253                 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
 254
 255                 // This is a hack, but it's an effective one. It allows us to do CSE on the
 256                 // primordial load of arguments. This assumes that the GetLocal that got put in
 257                 // place of the original SetArgument doesn't have any effects before it. This
 258                 // should hold true.
 259                 m_loadedArgumentValues.add(node, jsValue);
 260             }
 261
 262             switch (m_graph.m_argumentFormats[i]) {
 263             case FlushedInt32:
 264                 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
 265                 break;
 266             case FlushedBoolean:
 267                 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
 268                 break;
 269             case FlushedCell:
 270                 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
 271                 break;
 272             case FlushedJSValue:
 273                 break;
 274             default:
 275                 DFG_CRASH(m_graph, node, "Bad flush format for argument");
 276                 break;
 277             }
 278         }
 279         m_out.jump(lowBlock(m_graph.block(0)));
 280
 281         for (DFG::BasicBlock* block : preOrder)
 282             compileBlock(block);
 283
 284         // Make sure everything is decorated. This does a bunch of deferred decorating. This has
 285         // to happen last because our abstract heaps are generated lazily. They have to be
 286         // generated lazily because we have an infiniten number of numbered, indexed, and
 287         // absolute heaps. We only become aware of the ones we actually mention while lowering.
 288         m_heaps.computeRangesAndDecorateInstructions();
 289
 290         // We create all Phi's up front, but we may then decide not to compile the basic block
 291         // that would have contained one of them. So this creates orphans, which triggers B3
 292         // validation failures. Calling this fixes the issue.
 293         //
 294         // Note that you should avoid the temptation to make this call conditional upon
 295         // validation being enabled. B3 makes no guarantees of any kind of correctness when
 296         // dealing with IR that would have failed validation. For example, it would be valid to
 297         // write a B3 phase that so aggressively assumes the lack of orphans that it would crash
 298         // if any orphans were around. We might even have such phases already.
 299         m_proc.deleteOrphans();
 300
 301         // We put the blocks into the B3 procedure in a super weird order. Now we reorder them.
 302         m_out.applyBlockOrder();
 303     }
 304
 305 private:
 306
 307     void createPhiVariables()
 308     {
 309         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 310             DFG::BasicBlock* block = m_graph.block(blockIndex);
 311             if (!block)
 312                 continue;
 313             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 314                 Node* node = block->at(nodeIndex);
 315                 if (node->op() != DFG::Phi)
 316                     continue;
 317                 LType type;
 318                 switch (node->flags() & NodeResultMask) {
 319                 case NodeResultDouble:
 320                     type = m_out.doubleType;
 321                     break;
 322                 case NodeResultInt32:
 323                     type = m_out.int32;
 324                     break;
 325                 case NodeResultInt52:
 326                     type = m_out.int64;
 327                     break;
 328                 case NodeResultBoolean:
 329                     type = m_out.boolean;
 330                     break;
 331                 case NodeResultJS:
 332                     type = m_out.int64;
 333                     break;
 334                 default:
 335                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
 336                     break;
 337                 }
 338                 m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));
 339             }
 340         }
 341     }
 342
 343     void compileBlock(DFG::BasicBlock* block)
 344     {
 345         if (!block)
 346             return;
 347
 348         if (verboseCompilationEnabled())
 349             dataLog("Compiling block ", *block, "\n");
 350
 351         m_highBlock = block;
 352
 353         // Make sure that any blocks created while lowering code in the high block have the frequency of
 354         // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs
 355         // something roughly approximate for things like register allocation.
 356         m_out.setFrequency(m_highBlock->executionCount);
 357
 358         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 359
 360         m_nextHighBlock = 0;
 361         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 362             m_nextHighBlock = m_graph.block(nextBlockIndex);
 363             if (m_nextHighBlock)
 364                 break;
 365         }
 366         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 367
 368         // All of this effort to find the next block gives us the ability to keep the
 369         // generated IR in roughly program order. This ought not affect the performance
 370         // of the generated code (since we expect B3 to reorder things) but it will
 371         // make IR dumps easier to read.
 372         m_out.appendTo(lowBlock, m_nextLowBlock);
 373
 374         if (Options::ftlCrashes())
 375             m_out.trap();
 376
 377         if (!m_highBlock->cfaHasVisited) {
 378             if (verboseCompilationEnabled())
 379                 dataLog("Bailing because CFA didn't reach.\n");
 380             crash(m_highBlock->index, UINT_MAX);
 381             return;
 382         }
 383
 384         m_availabilityCalculator.beginBlock(m_highBlock);
 385
 386         m_state.reset();
 387         m_state.beginBasicBlock(m_highBlock);
 388
 389         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 390             if (!compileNode(m_nodeIndex))
 391                 break;
 392         }
 393     }
 394
 395     void safelyInvalidateAfterTermination()
 396     {
 397         if (verboseCompilationEnabled())
 398             dataLog("Bailing.\n");
 399         crash();
 400
 401         // Invalidate dominated blocks. Under normal circumstances we would expect
 402         // them to be invalidated already. But you can have the CFA become more
 403         // precise over time because the structures of objects change on the main
 404         // thread. Failing to do this would result in weird crashes due to a value
 405         // being used but not defined. Race conditions FTW!
 406         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 407             DFG::BasicBlock* target = m_graph.block(blockIndex);
 408             if (!target)
 409                 continue;
 410             if (m_graph.m_dominators->dominates(m_highBlock, target)) {
 411                 if (verboseCompilationEnabled())
 412                     dataLog("Block ", *target, " will bail also.\n");
 413                 target->cfaHasVisited = false;
 414             }
 415         }
 416     }
 417
 418     bool compileNode(unsigned nodeIndex)
 419     {
 420         if (!m_state.isValid()) {
 421             safelyInvalidateAfterTermination();
 422             return false;
 423         }
 424
 425         m_node = m_highBlock->at(nodeIndex);
 426         m_origin = m_node->origin;
 427         m_out.setOrigin(m_node);
 428
 429         if (verboseCompilationEnabled())
 430             dataLog("Lowering ", m_node, "\n");
 431
 432         m_availableRecoveries.resize(0);
 433
 434         m_interpreter.startExecuting();
 435
 436         switch (m_node->op()) {
 437         case DFG::Upsilon:
 438             compileUpsilon();
 439             break;
 440         case DFG::Phi:
 441             compilePhi();
 442             break;
 443         case JSConstant:
 444             break;
 445         case DoubleConstant:
 446             compileDoubleConstant();
 447             break;
 448         case Int52Constant:
 449             compileInt52Constant();
 450             break;
 451         case LazyJSConstant:
 452             compileLazyJSConstant();
 453             break;
 454         case DoubleRep:
 455             compileDoubleRep();
 456             break;
 457         case DoubleAsInt32:
 458             compileDoubleAsInt32();
 459             break;
 460         case DFG::ValueRep:
 461             compileValueRep();
 462             break;
 463         case Int52Rep:
 464             compileInt52Rep();
 465             break;
 466         case ValueToInt32:
 467             compileValueToInt32();
 468             break;
 469         case BooleanToNumber:
 470             compileBooleanToNumber();
 471             break;
 472         case ExtractOSREntryLocal:
 473             compileExtractOSREntryLocal();
 474             break;
 475         case GetStack:
 476             compileGetStack();
 477             break;
 478         case PutStack:
 479             compilePutStack();
 480             break;
 481         case DFG::Check:
 482             compileNoOp();
 483             break;
 484         case CallObjectConstructor:
 485             compileCallObjectConstructor();
 486             break;
 487         case ToThis:
 488             compileToThis();
 489             break;
 490         case ValueAdd:
 491             compileValueAdd();
 492             break;
 493         case StrCat:
 494             compileStrCat();
 495             break;
 496         case ArithAdd:
 497         case ArithSub:
 498             compileArithAddOrSub();
 499             break;
 500         case ArithClz32:
 501             compileArithClz32();
 502             break;
 503         case ArithMul:
 504             compileArithMul();
 505             break;
 506         case ArithDiv:
 507             compileArithDiv();
 508             break;
 509         case ArithMod:
 510             compileArithMod();
 511             break;
 512         case ArithMin:
 513         case ArithMax:
 514             compileArithMinOrMax();
 515             break;
 516         case ArithAbs:
 517             compileArithAbs();
 518             break;
 519         case ArithSin:
 520             compileArithSin();
 521             break;
 522         case ArithCos:
 523             compileArithCos();
 524             break;
 525         case ArithPow:
 526             compileArithPow();
 527             break;
 528         case ArithRandom:
 529             compileArithRandom();
 530             break;
 531         case ArithRound:
 532             compileArithRound();
 533             break;
 534         case ArithFloor:
 535             compileArithFloor();
 536             break;
 537         case ArithCeil:
 538             compileArithCeil();
 539             break;
 540         case ArithTrunc:
 541             compileArithTrunc();
 542             break;
 543         case ArithSqrt:
 544             compileArithSqrt();
 545             break;
 546         case ArithLog:
 547             compileArithLog();
 548             break;
 549         case ArithFRound:
 550             compileArithFRound();
 551             break;
 552         case ArithNegate:
 553             compileArithNegate();
 554             break;
 555         case DFG::BitAnd:
 556             compileBitAnd();
 557             break;
 558         case DFG::BitOr:
 559             compileBitOr();
 560             break;
 561         case DFG::BitXor:
 562             compileBitXor();
 563             break;
 564         case BitRShift:
 565             compileBitRShift();
 566             break;
 567         case BitLShift:
 568             compileBitLShift();
 569             break;
 570         case BitURShift:
 571             compileBitURShift();
 572             break;
 573         case UInt32ToNumber:
 574             compileUInt32ToNumber();
 575             break;
 576         case CheckStructure:
 577             compileCheckStructure();
 578             break;
 579         case CheckCell:
 580             compileCheckCell();
 581             break;
 582         case CheckNotEmpty:
 583             compileCheckNotEmpty();
 584             break;
 585         case CheckBadCell:
 586             compileCheckBadCell();
 587             break;
 588         case CheckIdent:
 589             compileCheckIdent();
 590             break;
 591         case GetExecutable:
 592             compileGetExecutable();
 593             break;
 594         case ArrayifyToStructure:
 595             compileArrayifyToStructure();
 596             break;
 597         case PutStructure:
 598             compilePutStructure();
 599             break;
 600         case TryGetById:
 601             compileGetById(AccessType::GetPure);
 602             break;
 603         case GetById:
 604         case GetByIdFlush:
 605             compileGetById(AccessType::Get);
 606             break;
 607         case In:
 608             compileIn();
 609             break;
 610         case PutById:
 611         case PutByIdDirect:
 612         case PutByIdFlush:
 613             compilePutById();
 614             break;
 615         case PutGetterById:
 616         case PutSetterById:
 617             compilePutAccessorById();
 618             break;
 619         case PutGetterSetterById:
 620             compilePutGetterSetterById();
 621             break;
 622         case PutGetterByVal:
 623         case PutSetterByVal:
 624             compilePutAccessorByVal();
 625             break;
 626         case GetButterfly:
 627             compileGetButterfly();
 628             break;
 629         case ConstantStoragePointer:
 630             compileConstantStoragePointer();
 631             break;
 632         case GetIndexedPropertyStorage:
 633             compileGetIndexedPropertyStorage();
 634             break;
 635         case CheckArray:
 636             compileCheckArray();
 637             break;
 638         case GetArrayLength:
 639             compileGetArrayLength();
 640             break;
 641         case CheckInBounds:
 642             compileCheckInBounds();
 643             break;
 644         case GetByVal:
 645             compileGetByVal();
 646             break;
 647         case GetMyArgumentByVal:
 648             compileGetMyArgumentByVal();
 649             break;
 650         case PutByVal:
 651         case PutByValAlias:
 652         case PutByValDirect:
 653             compilePutByVal();
 654             break;
 655         case ArrayPush:
 656             compileArrayPush();
 657             break;
 658         case ArrayPop:
 659             compileArrayPop();
 660             break;
 661         case CreateActivation:
 662             compileCreateActivation();
 663             break;
 664         case NewFunction:
 665         case NewGeneratorFunction:
 666             compileNewFunction();
 667             break;
 668         case CreateDirectArguments:
 669             compileCreateDirectArguments();
 670             break;
 671         case CreateScopedArguments:
 672             compileCreateScopedArguments();
 673             break;
 674         case CreateClonedArguments:
 675             compileCreateClonedArguments();
 676             break;
 677         case NewObject:
 678             compileNewObject();
 679             break;
 680         case NewArray:
 681             compileNewArray();
 682             break;
 683         case NewArrayBuffer:
 684             compileNewArrayBuffer();
 685             break;
 686         case NewArrayWithSize:
 687             compileNewArrayWithSize();
 688             break;
 689         case NewTypedArray:
 690             compileNewTypedArray();
 691             break;
 692         case GetTypedArrayByteOffset:
 693             compileGetTypedArrayByteOffset();
 694             break;
 695         case AllocatePropertyStorage:
 696             compileAllocatePropertyStorage();
 697             break;
 698         case ReallocatePropertyStorage:
 699             compileReallocatePropertyStorage();
 700             break;
 701         case ToString:
 702         case CallStringConstructor:
 703             compileToStringOrCallStringConstructor();
 704             break;
 705         case ToPrimitive:
 706             compileToPrimitive();
 707             break;
 708         case MakeRope:
 709             compileMakeRope();
 710             break;
 711         case StringCharAt:
 712             compileStringCharAt();
 713             break;
 714         case StringCharCodeAt:
 715             compileStringCharCodeAt();
 716             break;
 717         case StringFromCharCode:
 718             compileStringFromCharCode();
 719             break;
 720         case GetByOffset:
 721         case GetGetterSetterByOffset:
 722             compileGetByOffset();
 723             break;
 724         case GetGetter:
 725             compileGetGetter();
 726             break;
 727         case GetSetter:
 728             compileGetSetter();
 729             break;
 730         case MultiGetByOffset:
 731             compileMultiGetByOffset();
 732             break;
 733         case PutByOffset:
 734             compilePutByOffset();
 735             break;
 736         case MultiPutByOffset:
 737             compileMultiPutByOffset();
 738             break;
 739         case GetGlobalVar:
 740         case GetGlobalLexicalVariable:
 741             compileGetGlobalVariable();
 742             break;
 743         case PutGlobalVariable:
 744             compilePutGlobalVariable();
 745             break;
 746         case NotifyWrite:
 747             compileNotifyWrite();
 748             break;
 749         case GetCallee:
 750             compileGetCallee();
 751             break;
 752         case GetArgumentCount:
 753             compileGetArgumentCount();
 754             break;
 755         case GetScope:
 756             compileGetScope();
 757             break;
 758         case SkipScope:
 759             compileSkipScope();
 760             break;
 761         case GetGlobalObject:
 762             compileGetGlobalObject();
 763             break;
 764         case GetClosureVar:
 765             compileGetClosureVar();
 766             break;
 767         case PutClosureVar:
 768             compilePutClosureVar();
 769             break;
 770         case GetFromArguments:
 771             compileGetFromArguments();
 772             break;
 773         case PutToArguments:
 774             compilePutToArguments();
 775             break;
 776         case CompareEq:
 777             compileCompareEq();
 778             break;
 779         case CompareStrictEq:
 780             compileCompareStrictEq();
 781             break;
 782         case CompareLess:
 783             compileCompareLess();
 784             break;
 785         case CompareLessEq:
 786             compileCompareLessEq();
 787             break;
 788         case CompareGreater:
 789             compileCompareGreater();
 790             break;
 791         case CompareGreaterEq:
 792             compileCompareGreaterEq();
 793             break;
 794         case LogicalNot:
 795             compileLogicalNot();
 796             break;
 797         case Call:
 798         case TailCallInlinedCaller:
 799         case Construct:
 800             compileCallOrConstruct();
 801             break;
 802         case TailCall:
 803             compileTailCall();
 804             break;
 805         case CallVarargs:
 806         case CallForwardVarargs:
 807         case TailCallVarargs:
 808         case TailCallVarargsInlinedCaller:
 809         case TailCallForwardVarargs:
 810         case TailCallForwardVarargsInlinedCaller:
 811         case ConstructVarargs:
 812         case ConstructForwardVarargs:
 813             compileCallOrConstructVarargs();
 814             break;
 815         case LoadVarargs:
 816             compileLoadVarargs();
 817             break;
 818         case ForwardVarargs:
 819             compileForwardVarargs();
 820             break;
 821         case DFG::Jump:
 822             compileJump();
 823             break;
 824         case DFG::Branch:
 825             compileBranch();
 826             break;
 827         case DFG::Switch:
 828             compileSwitch();
 829             break;
 830         case DFG::Return:
 831             compileReturn();
 832             break;
 833         case ForceOSRExit:
 834             compileForceOSRExit();
 835             break;
 836         case Throw:
 837         case ThrowReferenceError:
 838             compileThrow();
 839             break;
 840         case InvalidationPoint:
 841             compileInvalidationPoint();
 842             break;
 843         case IsUndefined:
 844             compileIsUndefined();
 845             break;
 846         case IsBoolean:
 847             compileIsBoolean();
 848             break;
 849         case IsNumber:
 850             compileIsNumber();
 851             break;
 852         case IsString:
 853             compileIsString();
 854             break;
 855         case IsArrayObject:
 856             compileIsArrayObject();
 857             break;
 858         case IsJSArray:
 859             compileIsJSArray();
 860             break;
 861         case IsArrayConstructor:
 862             compileIsArrayConstructor();
 863             break;
 864         case IsObject:
 865             compileIsObject();
 866             break;
 867         case IsObjectOrNull:
 868             compileIsObjectOrNull();
 869             break;
 870         case IsFunction:
 871             compileIsFunction();
 872             break;
 873         case TypeOf:
 874             compileTypeOf();
 875             break;
 876         case CheckTypeInfoFlags:
 877             compileCheckTypeInfoFlags();
 878             break;
 879         case OverridesHasInstance:
 880             compileOverridesHasInstance();
 881             break;
 882         case InstanceOf:
 883             compileInstanceOf();
 884             break;
 885         case InstanceOfCustom:
 886             compileInstanceOfCustom();
 887             break;
 888         case CountExecution:
 889             compileCountExecution();
 890             break;
 891         case StoreBarrier:
 892             compileStoreBarrier();
 893             break;
 894         case HasIndexedProperty:
 895             compileHasIndexedProperty();
 896             break;
 897         case HasGenericProperty:
 898             compileHasGenericProperty();
 899             break;
 900         case HasStructureProperty:
 901             compileHasStructureProperty();
 902             break;
 903         case GetDirectPname:
 904             compileGetDirectPname();
 905             break;
 906         case GetEnumerableLength:
 907             compileGetEnumerableLength();
 908             break;
 909         case GetPropertyEnumerator:
 910             compileGetPropertyEnumerator();
 911             break;
 912         case GetEnumeratorStructurePname:
 913             compileGetEnumeratorStructurePname();
 914             break;
 915         case GetEnumeratorGenericPname:
 916             compileGetEnumeratorGenericPname();
 917             break;
 918         case ToIndexString:
 919             compileToIndexString();
 920             break;
 921         case CheckStructureImmediate:
 922             compileCheckStructureImmediate();
 923             break;
 924         case MaterializeNewObject:
 925             compileMaterializeNewObject();
 926             break;
 927         case MaterializeCreateActivation:
 928             compileMaterializeCreateActivation();
 929             break;
 930         case CheckWatchdogTimer:
 931             compileCheckWatchdogTimer();
 932             break;
 933         case CopyRest:
 934             compileCopyRest();
 935             break;
 936         case GetRestLength:
 937             compileGetRestLength();
 938             break;
 939         case RegExpExec:
 940             compileRegExpExec();
 941             break;
 942         case RegExpTest:
 943             compileRegExpTest();
 944             break;
 945         case NewRegexp:
 946             compileNewRegexp();
 947             break;
 948         case SetFunctionName:
 949             compileSetFunctionName();
 950             break;
 951         case StringReplace:
 952             compileStringReplace();
 953             break;
 954         case GetRegExpObjectLastIndex:
 955             compileGetRegExpObjectLastIndex();
 956             break;
 957         case SetRegExpObjectLastIndex:
 958             compileSetRegExpObjectLastIndex();
 959             break;
 960         case LogShadowChickenPrologue:
 961             compileLogShadowChickenPrologue();
 962             break;
 963         case LogShadowChickenTail:
 964             compileLogShadowChickenTail();
 965             break;
 966         case RecordRegExpCachedResult:
 967             compileRecordRegExpCachedResult();
 968             break;
 969
 970         case PhantomLocal:
 971         case LoopHint:
 972         case MovHint:
 973         case ZombieHint:
 974         case ExitOK:
 975         case PhantomNewObject:
 976         case PhantomNewFunction:
 977         case PhantomNewGeneratorFunction:
 978         case PhantomCreateActivation:
 979         case PhantomDirectArguments:
 980         case PhantomClonedArguments:
 981         case PutHint:
 982         case BottomValue:
 983         case KillStack:
 984             break;
 985         default:
 986             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
 987             break;
 988         }
 989
 990         if (m_node->isTerminal())
 991             return false;
 992
 993         if (!m_state.isValid()) {
 994             safelyInvalidateAfterTermination();
 995             return false;
 996         }
 997
 998         m_availabilityCalculator.executeNode(m_node);
 999         m_interpreter.executeEffects(nodeIndex);
1000
1001         return true;
1002     }
1003
1004     void compileUpsilon()
1005     {
1006         LValue upsilonValue = nullptr;
1007         switch (m_node->child1().useKind()) {
1008         case DoubleRepUse:
1009             upsilonValue = lowDouble(m_node->child1());
1010             break;
1011         case Int32Use:
1012         case KnownInt32Use:
1013             upsilonValue = lowInt32(m_node->child1());
1014             break;
1015         case Int52RepUse:
1016             upsilonValue = lowInt52(m_node->child1());
1017             break;
1018         case BooleanUse:
1019         case KnownBooleanUse:
1020             upsilonValue = lowBoolean(m_node->child1());
1021             break;
1022         case CellUse:
1023         case KnownCellUse:
1024             upsilonValue = lowCell(m_node->child1());
1025             break;
1026         case UntypedUse:
1027             upsilonValue = lowJSValue(m_node->child1());
1028             break;
1029         default:
1030             DFG_CRASH(m_graph, m_node, "Bad use kind");
1031             break;
1032         }
1033         ValueFromBlock upsilon = m_out.anchor(upsilonValue);
1034         LValue phiNode = m_phis.get(m_node->phi());
1035         m_out.addIncomingToPhi(phiNode, upsilon);
1036     }
1037
1038     void compilePhi()
1039     {
1040         LValue phi = m_phis.get(m_node);
1041         m_out.m_block->append(phi);
1042
1043         switch (m_node->flags() & NodeResultMask) {
1044         case NodeResultDouble:
1045             setDouble(phi);
1046             break;
1047         case NodeResultInt32:
1048             setInt32(phi);
1049             break;
1050         case NodeResultInt52:
1051             setInt52(phi);
1052             break;
1053         case NodeResultBoolean:
1054             setBoolean(phi);
1055             break;
1056         case NodeResultJS:
1057             setJSValue(phi);
1058             break;
1059         default:
1060             DFG_CRASH(m_graph, m_node, "Bad use kind");
1061             break;
1062         }
1063     }
1064
1065     void compileDoubleConstant()
1066     {
1067         setDouble(m_out.constDouble(m_node->asNumber()));
1068     }
1069
1070     void compileInt52Constant()
1071     {
1072         int64_t value = m_node->asMachineInt();
1073
1074         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
1075         setStrictInt52(m_out.constInt64(value));
1076     }
1077
1078     void compileLazyJSConstant()
1079     {
1080         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
1081         LazyJSValue value = m_node->lazyJSValue();
1082         patchpoint->setGenerator(
1083             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1084                 value.emit(jit, JSValueRegs(params[0].gpr()));
1085             });
1086         patchpoint->effects = Effects::none();
1087         setJSValue(patchpoint);
1088     }
1089
1090     void compileDoubleRep()
1091     {
1092         switch (m_node->child1().useKind()) {
1093         case RealNumberUse: {
1094             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1095
1096             LValue doubleValue = unboxDouble(value);
1097
1098             LBasicBlock intCase = m_out.newBlock();
1099             LBasicBlock continuation = m_out.newBlock();
1100
1101             ValueFromBlock fastResult = m_out.anchor(doubleValue);
1102             m_out.branch(
1103                 m_out.doubleEqual(doubleValue, doubleValue),
1104                 usually(continuation), rarely(intCase));
1105
1106             LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
1107
1108             FTL_TYPE_CHECK(
1109                 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
1110                 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
1111             ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
1112             m_out.jump(continuation);
1113
1114             m_out.appendTo(continuation, lastNext);
1115
1116             setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult));
1117             return;
1118         }
1119
1120         case NotCellUse:
1121         case NumberUse: {
1122             bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
1123
1124             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1125
1126             LBasicBlock intCase = m_out.newBlock();
1127             LBasicBlock doubleTesting = m_out.newBlock();
1128             LBasicBlock doubleCase = m_out.newBlock();
1129             LBasicBlock nonDoubleCase = m_out.newBlock();
1130             LBasicBlock continuation = m_out.newBlock();
1131
1132             m_out.branch(
1133                 isNotInt32(value, provenType(m_node->child1())),
1134                 unsure(doubleTesting), unsure(intCase));
1135
1136             LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1137
1138             ValueFromBlock intToDouble = m_out.anchor(
1139                 m_out.intToDouble(unboxInt32(value)));
1140             m_out.jump(continuation);
1141
1142             m_out.appendTo(doubleTesting, doubleCase);
1143             LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1144             m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1145
1146             m_out.appendTo(doubleCase, nonDoubleCase);
1147             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1148             m_out.jump(continuation);
1149
1150             if (shouldConvertNonNumber) {
1151                 LBasicBlock undefinedCase = m_out.newBlock();
1152                 LBasicBlock testNullCase = m_out.newBlock();
1153                 LBasicBlock nullCase = m_out.newBlock();
1154                 LBasicBlock testBooleanTrueCase = m_out.newBlock();
1155                 LBasicBlock convertBooleanTrueCase = m_out.newBlock();
1156                 LBasicBlock convertBooleanFalseCase = m_out.newBlock();
1157
1158                 m_out.appendTo(nonDoubleCase, undefinedCase);
1159                 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1160                 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1161
1162                 m_out.appendTo(undefinedCase, testNullCase);
1163                 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1164                 m_out.jump(continuation);
1165
1166                 m_out.appendTo(testNullCase, nullCase);
1167                 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1168                 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1169
1170                 m_out.appendTo(nullCase, testBooleanTrueCase);
1171                 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1172                 m_out.jump(continuation);
1173
1174                 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1175                 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1176                 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1177
1178                 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1179                 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1180                 m_out.jump(continuation);
1181
1182                 m_out.appendTo(convertBooleanFalseCase, continuation);
1183
1184                 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1185                 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse);
1186                 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1187                 m_out.jump(continuation);
1188
1189                 m_out.appendTo(continuation, lastNext);
1190                 setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1191                 return;
1192             }
1193             m_out.appendTo(nonDoubleCase, continuation);
1194             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1195             m_out.unreachable();
1196
1197             m_out.appendTo(continuation, lastNext);
1198
1199             setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble));
1200             return;
1201         }
1202
1203         case Int52RepUse: {
1204             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1205             return;
1206         }
1207
1208         default:
1209             DFG_CRASH(m_graph, m_node, "Bad use kind");
1210         }
1211     }
1212
1213     void compileDoubleAsInt32()
1214     {
1215         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1216         setInt32(integerValue);
1217     }
1218
1219     void compileValueRep()
1220     {
1221         switch (m_node->child1().useKind()) {
1222         case DoubleRepUse: {
1223             LValue value = lowDouble(m_node->child1());
1224
1225             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1226                 value = m_out.select(
1227                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1228             }
1229
1230             setJSValue(boxDouble(value));
1231             return;
1232         }
1233
1234         case Int52RepUse: {
1235             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1236             return;
1237         }
1238
1239         default:
1240             DFG_CRASH(m_graph, m_node, "Bad use kind");
1241         }
1242     }
1243
1244     void compileInt52Rep()
1245     {
1246         switch (m_node->child1().useKind()) {
1247         case Int32Use:
1248             setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1())));
1249             return;
1250
1251         case MachineIntUse:
1252             setStrictInt52(
1253                 jsValueToStrictInt52(
1254                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1255             return;
1256
1257         case DoubleRepMachineIntUse:
1258             setStrictInt52(
1259                 doubleToStrictInt52(
1260                     m_node->child1(), lowDouble(m_node->child1())));
1261             return;
1262
1263         default:
1264             RELEASE_ASSERT_NOT_REACHED();
1265         }
1266     }
1267
1268     void compileValueToInt32()
1269     {
1270         switch (m_node->child1().useKind()) {
1271         case Int52RepUse:
1272             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1273             break;
1274
1275         case DoubleRepUse:
1276             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1277             break;
1278
1279         case NumberUse:
1280         case NotCellUse: {
1281             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1282             if (isValid(value)) {
1283                 setInt32(value.value());
1284                 break;
1285             }
1286
1287             value = m_jsValueValues.get(m_node->child1().node());
1288             if (isValid(value)) {
1289                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1290                 break;
1291             }
1292
1293             // We'll basically just get here for constants. But it's good to have this
1294             // catch-all since we often add new representations into the mix.
1295             setInt32(
1296                 numberOrNotCellToInt32(
1297                     m_node->child1(),
1298                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1299             break;
1300         }
1301
1302         default:
1303             DFG_CRASH(m_graph, m_node, "Bad use kind");
1304             break;
1305         }
1306     }
1307
1308     void compileBooleanToNumber()
1309     {
1310         switch (m_node->child1().useKind()) {
1311         case BooleanUse: {
1312             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1313             return;
1314         }
1315
1316         case UntypedUse: {
1317             LValue value = lowJSValue(m_node->child1());
1318
1319             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1320                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1321                 return;
1322             }
1323
1324             LBasicBlock booleanCase = m_out.newBlock();
1325             LBasicBlock continuation = m_out.newBlock();
1326
1327             ValueFromBlock notBooleanResult = m_out.anchor(value);
1328             m_out.branch(
1329                 isBoolean(value, provenType(m_node->child1())),
1330                 unsure(booleanCase), unsure(continuation));
1331
1332             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1333             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1334                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1335             m_out.jump(continuation);
1336
1337             m_out.appendTo(continuation, lastNext);
1338             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1339             return;
1340         }
1341
1342         default:
1343             RELEASE_ASSERT_NOT_REACHED();
1344             return;
1345         }
1346     }
1347
1348     void compileExtractOSREntryLocal()
1349     {
1350         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1351             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1352         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1353     }
1354
1355     void compileGetStack()
1356     {
1357         // GetLocals arise only for captured variables and arguments. For arguments, we might have
1358         // already loaded it.
1359         if (LValue value = m_loadedArgumentValues.get(m_node)) {
1360             setJSValue(value);
1361             return;
1362         }
1363
1364         StackAccessData* data = m_node->stackAccessData();
1365         AbstractValue& value = m_state.variables().operand(data->local);
1366
1367         DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1368         DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1369
1370         if (isInt32Speculation(value.m_type))
1371             setInt32(m_out.load32(payloadFor(data->machineLocal)));
1372         else
1373             setJSValue(m_out.load64(addressFor(data->machineLocal)));
1374     }
1375
1376     void compilePutStack()
1377     {
1378         StackAccessData* data = m_node->stackAccessData();
1379         switch (data->format) {
1380         case FlushedJSValue: {
1381             LValue value = lowJSValue(m_node->child1());
1382             m_out.store64(value, addressFor(data->machineLocal));
1383             break;
1384         }
1385
1386         case FlushedDouble: {
1387             LValue value = lowDouble(m_node->child1());
1388             m_out.storeDouble(value, addressFor(data->machineLocal));
1389             break;
1390         }
1391
1392         case FlushedInt32: {
1393             LValue value = lowInt32(m_node->child1());
1394             m_out.store32(value, payloadFor(data->machineLocal));
1395             break;
1396         }
1397
1398         case FlushedInt52: {
1399             LValue value = lowInt52(m_node->child1());
1400             m_out.store64(value, addressFor(data->machineLocal));
1401             break;
1402         }
1403
1404         case FlushedCell: {
1405             LValue value = lowCell(m_node->child1());
1406             m_out.store64(value, addressFor(data->machineLocal));
1407             break;
1408         }
1409
1410         case FlushedBoolean: {
1411             speculateBoolean(m_node->child1());
1412             m_out.store64(
1413                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1414                 addressFor(data->machineLocal));
1415             break;
1416         }
1417
1418         default:
1419             DFG_CRASH(m_graph, m_node, "Bad flush format");
1420             break;
1421         }
1422     }
1423
1424     void compileNoOp()
1425     {
1426         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1427     }
1428
1429     void compileCallObjectConstructor()
1430     {
1431         LValue value = lowJSValue(m_node->child1());
1432
1433         LBasicBlock isCellCase = m_out.newBlock();
1434         LBasicBlock slowCase = m_out.newBlock();
1435         LBasicBlock continuation = m_out.newBlock();
1436
1437         m_out.branch(isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1438
1439         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1440         ValueFromBlock fastResult = m_out.anchor(value);
1441         m_out.branch(isObject(value), usually(continuation), rarely(slowCase));
1442
1443         m_out.appendTo(slowCase, continuation);
1444         ValueFromBlock slowResult = m_out.anchor(vmCall(m_out.int64, m_out.operation(operationToObject), m_callFrame, value));
1445         m_out.jump(continuation);
1446
1447         m_out.appendTo(continuation, lastNext);
1448         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1449     }
1450
1451     void compileToThis()
1452     {
1453         LValue value = lowJSValue(m_node->child1());
1454
1455         LBasicBlock isCellCase = m_out.newBlock();
1456         LBasicBlock slowCase = m_out.newBlock();
1457         LBasicBlock continuation = m_out.newBlock();
1458
1459         m_out.branch(
1460             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1461
1462         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1463         ValueFromBlock fastResult = m_out.anchor(value);
1464         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1465
1466         m_out.appendTo(slowCase, continuation);
1467         J_JITOperation_EJ function;
1468         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1469             function = operationToThisStrict;
1470         else
1471             function = operationToThis;
1472         ValueFromBlock slowResult = m_out.anchor(
1473             vmCall(m_out.int64, m_out.operation(function), m_callFrame, value));
1474         m_out.jump(continuation);
1475
1476         m_out.appendTo(continuation, lastNext);
1477         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1478     }
1479
1480     void compileValueAdd()
1481     {
1482         emitBinarySnippet<JITAddGenerator>(operationValueAdd);
1483     }
1484
1485     void compileStrCat()
1486     {
1487         LValue result;
1488         if (m_node->child3()) {
1489             result = vmCall(
1490                 m_out.int64, m_out.operation(operationStrCat3), m_callFrame,
1491                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1492                 lowJSValue(m_node->child2(), ManualOperandSpeculation),
1493                 lowJSValue(m_node->child3(), ManualOperandSpeculation));
1494         } else {
1495             result = vmCall(
1496                 m_out.int64, m_out.operation(operationStrCat2), m_callFrame,
1497                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1498                 lowJSValue(m_node->child2(), ManualOperandSpeculation));
1499         }
1500         setJSValue(result);
1501     }
1502
1503     void compileArithAddOrSub()
1504     {
1505         bool isSub =  m_node->op() == ArithSub;
1506         switch (m_node->binaryUseKind()) {
1507         case Int32Use: {
1508             LValue left = lowInt32(m_node->child1());
1509             LValue right = lowInt32(m_node->child2());
1510
1511             if (!shouldCheckOverflow(m_node->arithMode())) {
1512                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1513                 break;
1514             }
1515
1516             CheckValue* result =
1517                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
1518             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1519             setInt32(result);
1520             break;
1521         }
1522
1523         case Int52RepUse: {
1524             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)
1525                 && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) {
1526                 Int52Kind kind;
1527                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1528                 LValue right = lowInt52(m_node->child2(), kind);
1529                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1530                 break;
1531             }
1532
1533             LValue left = lowInt52(m_node->child1());
1534             LValue right = lowInt52(m_node->child2());
1535             CheckValue* result =
1536                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
1537             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1538             setInt52(result);
1539             break;
1540         }
1541
1542         case DoubleRepUse: {
1543             LValue C1 = lowDouble(m_node->child1());
1544             LValue C2 = lowDouble(m_node->child2());
1545
1546             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1547             break;
1548         }
1549
1550         case UntypedUse: {
1551             if (!isSub) {
1552                 DFG_CRASH(m_graph, m_node, "Bad use kind");
1553                 break;
1554             }
1555
1556             emitBinarySnippet<JITSubGenerator>(operationValueSub);
1557             break;
1558         }
1559
1560         default:
1561             DFG_CRASH(m_graph, m_node, "Bad use kind");
1562             break;
1563         }
1564     }
1565
1566     void compileArithClz32()
1567     {
1568         LValue operand = lowInt32(m_node->child1());
1569         setInt32(m_out.ctlz32(operand));
1570     }
1571
1572     void compileArithMul()
1573     {
1574         switch (m_node->binaryUseKind()) {
1575         case Int32Use: {
1576             LValue left = lowInt32(m_node->child1());
1577             LValue right = lowInt32(m_node->child2());
1578
1579             LValue result;
1580
1581             if (!shouldCheckOverflow(m_node->arithMode()))
1582                 result = m_out.mul(left, right);
1583             else {
1584                 CheckValue* speculation = m_out.speculateMul(left, right);
1585                 blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin);
1586                 result = speculation;
1587             }
1588
1589             if (shouldCheckNegativeZero(m_node->arithMode())) {
1590                 LBasicBlock slowCase = m_out.newBlock();
1591                 LBasicBlock continuation = m_out.newBlock();
1592
1593                 m_out.branch(
1594                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1595
1596                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1597                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero));
1598                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero));
1599                 m_out.jump(continuation);
1600                 m_out.appendTo(continuation, lastNext);
1601             }
1602
1603             setInt32(result);
1604             break;
1605         }
1606
1607         case Int52RepUse: {
1608             Int52Kind kind;
1609             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1610             LValue right = lowInt52(m_node->child2(), opposite(kind));
1611
1612             CheckValue* result = m_out.speculateMul(left, right);
1613             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1614
1615             if (shouldCheckNegativeZero(m_node->arithMode())) {
1616                 LBasicBlock slowCase = m_out.newBlock();
1617                 LBasicBlock continuation = m_out.newBlock();
1618
1619                 m_out.branch(
1620                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1621
1622                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1623                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero));
1624                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero));
1625                 m_out.jump(continuation);
1626                 m_out.appendTo(continuation, lastNext);
1627             }
1628
1629             setInt52(result);
1630             break;
1631         }
1632
1633         case DoubleRepUse: {
1634             setDouble(
1635                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1636             break;
1637         }
1638
1639         case UntypedUse: {
1640             emitBinarySnippet<JITMulGenerator>(operationValueMul);
1641             break;
1642         }
1643
1644         default:
1645             DFG_CRASH(m_graph, m_node, "Bad use kind");
1646             break;
1647         }
1648     }
1649
1650     void compileArithDiv()
1651     {
1652         switch (m_node->binaryUseKind()) {
1653         case Int32Use: {
1654             LValue numerator = lowInt32(m_node->child1());
1655             LValue denominator = lowInt32(m_node->child2());
1656
1657             if (shouldCheckNegativeZero(m_node->arithMode())) {
1658                 LBasicBlock zeroNumerator = m_out.newBlock();
1659                 LBasicBlock numeratorContinuation = m_out.newBlock();
1660
1661                 m_out.branch(
1662                     m_out.isZero32(numerator),
1663                     rarely(zeroNumerator), usually(numeratorContinuation));
1664
1665                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1666
1667                 speculate(
1668                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1669
1670                 m_out.jump(numeratorContinuation);
1671
1672                 m_out.appendTo(numeratorContinuation, innerLastNext);
1673             }
1674
1675             if (shouldCheckOverflow(m_node->arithMode())) {
1676                 LBasicBlock unsafeDenominator = m_out.newBlock();
1677                 LBasicBlock continuation = m_out.newBlock();
1678
1679                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1680                 m_out.branch(
1681                     m_out.above(adjustedDenominator, m_out.int32One),
1682                     usually(continuation), rarely(unsafeDenominator));
1683
1684                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1685                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1686                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
1687                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
1688                 m_out.jump(continuation);
1689
1690                 m_out.appendTo(continuation, lastNext);
1691                 LValue result = m_out.div(numerator, denominator);
1692                 speculate(
1693                     Overflow, noValue(), 0,
1694                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1695                 setInt32(result);
1696             } else
1697                 setInt32(m_out.chillDiv(numerator, denominator));
1698
1699             break;
1700         }
1701
1702         case DoubleRepUse: {
1703             setDouble(m_out.doubleDiv(
1704                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1705             break;
1706         }
1707
1708         case UntypedUse: {
1709             emitBinarySnippet<JITDivGenerator, NeedScratchFPR>(operationValueDiv);
1710             break;
1711         }
1712
1713         default:
1714             DFG_CRASH(m_graph, m_node, "Bad use kind");
1715             break;
1716         }
1717     }
1718
1719     void compileArithMod()
1720     {
1721         switch (m_node->binaryUseKind()) {
1722         case Int32Use: {
1723             LValue numerator = lowInt32(m_node->child1());
1724             LValue denominator = lowInt32(m_node->child2());
1725
1726             LValue remainder;
1727             if (shouldCheckOverflow(m_node->arithMode())) {
1728                 LBasicBlock unsafeDenominator = m_out.newBlock();
1729                 LBasicBlock continuation = m_out.newBlock();
1730
1731                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1732                 m_out.branch(
1733                     m_out.above(adjustedDenominator, m_out.int32One),
1734                     usually(continuation), rarely(unsafeDenominator));
1735
1736                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1737                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1738                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
1739                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
1740                 m_out.jump(continuation);
1741
1742                 m_out.appendTo(continuation, lastNext);
1743                 LValue result = m_out.mod(numerator, denominator);
1744                 remainder = result;
1745             } else
1746                 remainder = m_out.chillMod(numerator, denominator);
1747
1748             if (shouldCheckNegativeZero(m_node->arithMode())) {
1749                 LBasicBlock negativeNumerator = m_out.newBlock();
1750                 LBasicBlock numeratorContinuation = m_out.newBlock();
1751
1752                 m_out.branch(
1753                     m_out.lessThan(numerator, m_out.int32Zero),
1754                     unsure(negativeNumerator), unsure(numeratorContinuation));
1755
1756                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1757
1758                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1759
1760                 m_out.jump(numeratorContinuation);
1761
1762                 m_out.appendTo(numeratorContinuation, innerLastNext);
1763             }
1764
1765             setInt32(remainder);
1766             break;
1767         }
1768
1769         case DoubleRepUse: {
1770             setDouble(
1771                 m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1772             break;
1773         }
1774
1775         default:
1776             DFG_CRASH(m_graph, m_node, "Bad use kind");
1777             break;
1778         }
1779     }
1780
1781     void compileArithMinOrMax()
1782     {
1783         switch (m_node->binaryUseKind()) {
1784         case Int32Use: {
1785             LValue left = lowInt32(m_node->child1());
1786             LValue right = lowInt32(m_node->child2());
1787
1788             setInt32(
1789                 m_out.select(
1790                     m_node->op() == ArithMin
1791                         ? m_out.lessThan(left, right)
1792                         : m_out.lessThan(right, left),
1793                     left, right));
1794             break;
1795         }
1796
1797         case DoubleRepUse: {
1798             LValue left = lowDouble(m_node->child1());
1799             LValue right = lowDouble(m_node->child2());
1800
1801             LBasicBlock notLessThan = m_out.newBlock();
1802             LBasicBlock continuation = m_out.newBlock();
1803
1804             Vector<ValueFromBlock, 2> results;
1805
1806             results.append(m_out.anchor(left));
1807             m_out.branch(
1808                 m_node->op() == ArithMin
1809                     ? m_out.doubleLessThan(left, right)
1810                     : m_out.doubleGreaterThan(left, right),
1811                 unsure(continuation), unsure(notLessThan));
1812
1813             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1814             results.append(m_out.anchor(m_out.select(
1815                 m_node->op() == ArithMin
1816                     ? m_out.doubleGreaterThanOrEqual(left, right)
1817                     : m_out.doubleLessThanOrEqual(left, right),
1818                 right, m_out.constDouble(PNaN))));
1819             m_out.jump(continuation);
1820
1821             m_out.appendTo(continuation, lastNext);
1822             setDouble(m_out.phi(m_out.doubleType, results));
1823             break;
1824         }
1825
1826         default:
1827             DFG_CRASH(m_graph, m_node, "Bad use kind");
1828             break;
1829         }
1830     }
1831
1832     void compileArithAbs()
1833     {
1834         switch (m_node->child1().useKind()) {
1835         case Int32Use: {
1836             LValue value = lowInt32(m_node->child1());
1837
1838             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1839             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1840
1841             if (shouldCheckOverflow(m_node->arithMode()))
1842                 speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
1843
1844             setInt32(result);
1845             break;
1846         }
1847
1848         case DoubleRepUse: {
1849             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1850             break;
1851         }
1852
1853         default:
1854             DFG_CRASH(m_graph, m_node, "Bad use kind");
1855             break;
1856         }
1857     }
1858
1859     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1860
1861     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1862
1863     void compileArithPow()
1864     {
1865         if (m_node->child2().useKind() == Int32Use)
1866             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1867         else {
1868             LValue base = lowDouble(m_node->child1());
1869             LValue exponent = lowDouble(m_node->child2());
1870
1871             LBasicBlock integerExponentIsSmallBlock = m_out.newBlock();
1872             LBasicBlock integerExponentPowBlock = m_out.newBlock();
1873             LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock();
1874             LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock();
1875             LBasicBlock nanExceptionBaseIsOne = m_out.newBlock();
1876             LBasicBlock powBlock = m_out.newBlock();
1877             LBasicBlock nanExceptionResultIsNaN = m_out.newBlock();
1878             LBasicBlock continuation = m_out.newBlock();
1879
1880             LValue integerExponent = m_out.doubleToInt(exponent);
1881             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1882             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1883             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1884
1885             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1886             LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1887             m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1888
1889             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1890             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1891             m_out.jump(continuation);
1892
1893             // If y is NaN, the result is NaN.
1894             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1895             LValue exponentIsNaN;
1896             if (provenType(m_node->child2()) & SpecDoubleNaN)
1897                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1898             else
1899                 exponentIsNaN = m_out.booleanFalse;
1900             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1901
1902             // If abs(x) is 1 and y is +infinity, the result is NaN.
1903             // If abs(x) is 1 and y is -infinity, the result is NaN.
1904             m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1905             LValue absoluteExponent = m_out.doubleAbs(exponent);
1906             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1907             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1908
1909             m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1910             LValue absoluteBase = m_out.doubleAbs(base);
1911             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1912             m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1913
1914             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1915             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1916             m_out.jump(continuation);
1917
1918             m_out.appendTo(nanExceptionResultIsNaN, continuation);
1919             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1920             m_out.jump(continuation);
1921
1922             m_out.appendTo(continuation, lastNext);
1923             setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1924         }
1925     }
1926
1927     void compileArithRandom()
1928     {
1929         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
1930
1931         // Inlined WeakRandom::advance().
1932         // uint64_t x = m_low;
1933         void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
1934         LValue low = m_out.load64(m_out.absolute(lowAddress));
1935         // uint64_t y = m_high;
1936         void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();
1937         LValue high = m_out.load64(m_out.absolute(highAddress));
1938         // m_low = y;
1939         m_out.store64(high, m_out.absolute(lowAddress));
1940
1941         // x ^= x << 23;
1942         LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low);
1943
1944         // x ^= x >> 17;
1945         LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1);
1946
1947         // x ^= y ^ (y >> 26);
1948         LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2);
1949
1950         // m_high = x;
1951         m_out.store64(phase3, m_out.absolute(highAddress));
1952
1953         // return x + y;
1954         LValue random64 = m_out.add(phase3, high);
1955
1956         // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
1957         LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1));
1958
1959         LValue double53Integer = m_out.intToDouble(random53);
1960
1961         // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
1962         // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
1963         static const double scale = 1.0 / (1ULL << 53);
1964
1965         // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
1966         // It just reduces the exp part of the given 53bit double integer.
1967         // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
1968         // Now we get 53bit precision random double value in [0, 1).
1969         LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale));
1970
1971         setDouble(result);
1972     }
1973
1974     void compileArithRound()
1975     {
1976         LValue result = nullptr;
1977
1978         if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) {
1979             LValue value = lowDouble(m_node->child1());
1980             result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5)));
1981         } else {
1982             LBasicBlock realPartIsMoreThanHalf = m_out.newBlock();
1983             LBasicBlock continuation = m_out.newBlock();
1984
1985             LValue value = lowDouble(m_node->child1());
1986             LValue integerValue = m_out.doubleCeil(value);
1987             ValueFromBlock integerValueResult = m_out.anchor(integerValue);
1988
1989             LValue realPart = m_out.doubleSub(integerValue, value);
1990
1991             m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
1992
1993             LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
1994             LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
1995             ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
1996             m_out.jump(continuation);
1997             m_out.appendTo(continuation, lastNext);
1998
1999             result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult);
2000         }
2001
2002         if (producesInteger(m_node->arithRoundingMode())) {
2003             LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
2004             setInt32(integerValue);
2005         } else
2006             setDouble(result);
2007     }
2008
2009     void compileArithFloor()
2010     {
2011         LValue value = lowDouble(m_node->child1());
2012         LValue integerValue = m_out.doubleFloor(value);
2013         if (producesInteger(m_node->arithRoundingMode()))
2014             setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2015         else
2016             setDouble(integerValue);
2017     }
2018
2019     void compileArithCeil()
2020     {
2021         LValue value = lowDouble(m_node->child1());
2022         LValue integerValue = m_out.doubleCeil(value);
2023         if (producesInteger(m_node->arithRoundingMode()))
2024             setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2025         else
2026             setDouble(integerValue);
2027     }
2028
2029     void compileArithTrunc()
2030     {
2031         LValue value = lowDouble(m_node->child1());
2032         LValue result = m_out.doubleTrunc(value);
2033         if (producesInteger(m_node->arithRoundingMode()))
2034             setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2035         else
2036             setDouble(result);
2037     }
2038
2039     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
2040
2041     void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
2042
2043     void compileArithFRound()
2044     {
2045         setDouble(m_out.fround(lowDouble(m_node->child1())));
2046     }
2047
2048     void compileArithNegate()
2049     {
2050         switch (m_node->child1().useKind()) {
2051         case Int32Use: {
2052             LValue value = lowInt32(m_node->child1());
2053
2054             LValue result;
2055             if (!shouldCheckOverflow(m_node->arithMode()))
2056                 result = m_out.neg(value);
2057             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
2058                 CheckValue* check = m_out.speculateSub(m_out.int32Zero, value);
2059                 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
2060                 result = check;
2061             } else {
2062                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
2063                 result = m_out.neg(value);
2064             }
2065
2066             setInt32(result);
2067             break;
2068         }
2069
2070         case Int52RepUse: {
2071             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) {
2072                 Int52Kind kind;
2073                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
2074                 LValue result = m_out.neg(value);
2075                 if (shouldCheckNegativeZero(m_node->arithMode()))
2076                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
2077                 setInt52(result, kind);
2078                 break;
2079             }
2080
2081             LValue value = lowInt52(m_node->child1());
2082             CheckValue* result = m_out.speculateSub(m_out.int64Zero, value);
2083             blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin);
2084             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
2085             setInt52(result);
2086             break;
2087         }
2088
2089         case DoubleRepUse: {
2090             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
2091             break;
2092         }
2093
2094         default:
2095             DFG_CRASH(m_graph, m_node, "Bad use kind");
2096             break;
2097         }
2098     }
2099
2100     void compileBitAnd()
2101     {
2102         if (m_node->isBinaryUseKind(UntypedUse)) {
2103             emitBinaryBitOpSnippet<JITBitAndGenerator>(operationValueBitAnd);
2104             return;
2105         }
2106         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2107     }
2108
2109     void compileBitOr()
2110     {
2111         if (m_node->isBinaryUseKind(UntypedUse)) {
2112             emitBinaryBitOpSnippet<JITBitOrGenerator>(operationValueBitOr);
2113             return;
2114         }
2115         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2116     }
2117
2118     void compileBitXor()
2119     {
2120         if (m_node->isBinaryUseKind(UntypedUse)) {
2121             emitBinaryBitOpSnippet<JITBitXorGenerator>(operationValueBitXor);
2122             return;
2123         }
2124         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2125     }
2126
2127     void compileBitRShift()
2128     {
2129         if (m_node->isBinaryUseKind(UntypedUse)) {
2130             emitRightShiftSnippet(JITRightShiftGenerator::SignedShift);
2131             return;
2132         }
2133         setInt32(m_out.aShr(
2134             lowInt32(m_node->child1()),
2135             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2136     }
2137
2138     void compileBitLShift()
2139     {
2140         if (m_node->isBinaryUseKind(UntypedUse)) {
2141             emitBinaryBitOpSnippet<JITLeftShiftGenerator>(operationValueBitLShift);
2142             return;
2143         }
2144         setInt32(m_out.shl(
2145             lowInt32(m_node->child1()),
2146             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2147     }
2148
2149     void compileBitURShift()
2150     {
2151         if (m_node->isBinaryUseKind(UntypedUse)) {
2152             emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift);
2153             return;
2154         }
2155         setInt32(m_out.lShr(
2156             lowInt32(m_node->child1()),
2157             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2158     }
2159
2160     void compileUInt32ToNumber()
2161     {
2162         LValue value = lowInt32(m_node->child1());
2163
2164         if (doesOverflow(m_node->arithMode())) {
2165             setStrictInt52(m_out.zeroExtPtr(value));
2166             return;
2167         }
2168
2169         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
2170         setInt32(value);
2171     }
2172
2173     void compileCheckStructure()
2174     {
2175         ExitKind exitKind;
2176         if (m_node->child1()->hasConstant())
2177             exitKind = BadConstantCache;
2178         else
2179             exitKind = BadCache;
2180
2181         switch (m_node->child1().useKind()) {
2182         case CellUse:
2183         case KnownCellUse: {
2184             LValue cell = lowCell(m_node->child1());
2185
2186             checkStructure(
2187                 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
2188                 exitKind, m_node->structureSet(),
2189                 [&] (Structure* structure) {
2190                     return weakStructureID(structure);
2191                 });
2192             return;
2193         }
2194
2195         case CellOrOtherUse: {
2196             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
2197
2198             LBasicBlock cellCase = m_out.newBlock();
2199             LBasicBlock notCellCase = m_out.newBlock();
2200             LBasicBlock continuation = m_out.newBlock();
2201
2202             m_out.branch(
2203                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2204
2205             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2206             checkStructure(
2207                 m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value),
2208                 exitKind, m_node->structureSet(),
2209                 [&] (Structure* structure) {
2210                     return weakStructureID(structure);
2211                 });
2212             m_out.jump(continuation);
2213
2214             m_out.appendTo(notCellCase, continuation);
2215             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value));
2216             m_out.jump(continuation);
2217
2218             m_out.appendTo(continuation, lastNext);
2219             return;
2220         }
2221
2222         default:
2223             DFG_CRASH(m_graph, m_node, "Bad use kind");
2224             return;
2225         }
2226     }
2227
2228     void compileCheckCell()
2229     {
2230         LValue cell = lowCell(m_node->child1());
2231
2232         speculate(
2233             BadCell, jsValueValue(cell), m_node->child1().node(),
2234             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
2235     }
2236
2237     void compileCheckBadCell()
2238     {
2239         terminate(BadCell);
2240     }
2241
2242     void compileCheckNotEmpty()
2243     {
2244         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
2245     }
2246
2247     void compileCheckIdent()
2248     {
2249         UniquedStringImpl* uid = m_node->uidOperand();
2250         if (uid->isSymbol()) {
2251             LValue symbol = lowSymbol(m_node->child1());
2252             LValue stringImpl = m_out.loadPtr(symbol, m_heaps.Symbol_privateName);
2253             speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
2254         } else {
2255             LValue string = lowStringIdent(m_node->child1());
2256             LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
2257             speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
2258         }
2259     }
2260
2261     void compileGetExecutable()
2262     {
2263         LValue cell = lowCell(m_node->child1());
2264         speculateFunction(m_node->child1(), cell);
2265         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
2266     }
2267
2268     void compileArrayifyToStructure()
2269     {
2270         LValue cell = lowCell(m_node->child1());
2271         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
2272
2273         LBasicBlock unexpectedStructure = m_out.newBlock();
2274         LBasicBlock continuation = m_out.newBlock();
2275
2276         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2277
2278         m_out.branch(
2279             m_out.notEqual(structureID, weakStructureID(m_node->structure())),
2280             rarely(unexpectedStructure), usually(continuation));
2281
2282         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
2283
2284         if (property) {
2285             switch (m_node->arrayMode().type()) {
2286             case Array::Int32:
2287             case Array::Double:
2288             case Array::Contiguous:
2289                 speculate(
2290                     Uncountable, noValue(), 0,
2291                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
2292                 break;
2293             default:
2294                 break;
2295             }
2296         }
2297
2298         switch (m_node->arrayMode().type()) {
2299         case Array::Int32:
2300             vmCall(m_out.voidType, m_out.operation(operationEnsureInt32), m_callFrame, cell);
2301             break;
2302         case Array::Double:
2303             vmCall(m_out.voidType, m_out.operation(operationEnsureDouble), m_callFrame, cell);
2304             break;
2305         case Array::Contiguous:
2306             vmCall(m_out.voidType, m_out.operation(operationEnsureContiguous), m_callFrame, cell);
2307             break;
2308         case Array::ArrayStorage:
2309         case Array::SlowPutArrayStorage:
2310             vmCall(m_out.voidType, m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
2311             break;
2312         default:
2313             DFG_CRASH(m_graph, m_node, "Bad array type");
2314             break;
2315         }
2316
2317         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2318         speculate(
2319             BadIndexingType, jsValueValue(cell), 0,
2320             m_out.notEqual(structureID, weakStructureID(m_node->structure())));
2321         m_out.jump(continuation);
2322
2323         m_out.appendTo(continuation, lastNext);
2324     }
2325
2326     void compilePutStructure()
2327     {
2328         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
2329
2330         Structure* oldStructure = m_node->transition()->previous;
2331         Structure* newStructure = m_node->transition()->next;
2332         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
2333         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
2334         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
2335
2336         LValue cell = lowCell(m_node->child1());
2337         m_out.store32(
2338             weakStructureID(newStructure),
2339             cell, m_heaps.JSCell_structureID);
2340     }
2341
2342     void compileGetById(AccessType type)
2343     {
2344         ASSERT(type == AccessType::Get || type == AccessType::GetPure);
2345         switch (m_node->child1().useKind()) {
2346         case CellUse: {
2347             setJSValue(getById(lowCell(m_node->child1()), type));
2348             return;
2349         }
2350
2351         case UntypedUse: {
2352             // This is pretty weird, since we duplicate the slow path both here and in the
2353             // code generated by the IC. We should investigate making this less bad.
2354             // https://bugs.webkit.org/show_bug.cgi?id=127830
2355             LValue value = lowJSValue(m_node->child1());
2356
2357             LBasicBlock cellCase = m_out.newBlock();
2358             LBasicBlock notCellCase = m_out.newBlock();
2359             LBasicBlock continuation = m_out.newBlock();
2360
2361             m_out.branch(
2362                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2363
2364             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2365             ValueFromBlock cellResult = m_out.anchor(getById(value, type));
2366             m_out.jump(continuation);
2367
2368             J_JITOperation_EJI getByIdFunction;
2369             if (type == AccessType::Get)
2370                 getByIdFunction = operationGetByIdGeneric;
2371             else
2372                 getByIdFunction = operationTryGetByIdGeneric;
2373
2374             m_out.appendTo(notCellCase, continuation);
2375             ValueFromBlock notCellResult = m_out.anchor(vmCall(
2376                 m_out.int64, m_out.operation(getByIdFunction),
2377                 m_callFrame, value,
2378                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2379             m_out.jump(continuation);
2380
2381             m_out.appendTo(continuation, lastNext);
2382             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2383             return;
2384         }
2385
2386         default:
2387             DFG_CRASH(m_graph, m_node, "Bad use kind");
2388             return;
2389         }
2390     }
2391
2392     void compilePutById()
2393     {
2394         Node* node = m_node;
2395
2396         // See above; CellUse is easier so we do only that for now.
2397         ASSERT(node->child1().useKind() == CellUse);
2398
2399         LValue base = lowCell(node->child1());
2400         LValue value = lowJSValue(node->child2());
2401         auto uid = m_graph.identifiers()[node->identifierNumber()];
2402
2403         B3::PatchpointValue* patchpoint = m_out.patchpoint(Void);
2404         patchpoint->appendSomeRegister(base);
2405         patchpoint->appendSomeRegister(value);
2406         patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister));
2407         patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister));
2408         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2409
2410         // FIXME: If this is a PutByIdFlush, we might want to late-clobber volatile registers.
2411         // https://bugs.webkit.org/show_bug.cgi?id=152848
2412
2413         RefPtr<PatchpointExceptionHandle> exceptionHandle =
2414             preparePatchpointForExceptions(patchpoint);
2415
2416         State* state = &m_ftlState;
2417         ECMAMode ecmaMode = m_graph.executableFor(node->origin.semantic)->ecmaMode();
2418
2419         patchpoint->setGenerator(
2420             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2421                 AllowMacroScratchRegisterUsage allowScratch(jit);
2422
2423                 CallSiteIndex callSiteIndex =
2424                     state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
2425
2426                 Box<CCallHelpers::JumpList> exceptions =
2427                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2428
2429                 // JS setter call ICs generated by the PutById IC will need this.
2430                 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
2431
2432                 auto generator = Box<JITPutByIdGenerator>::create(
2433                     jit.codeBlock(), node->origin.semantic, callSiteIndex,
2434                     params.unavailableRegisters(), JSValueRegs(params[0].gpr()),
2435                     JSValueRegs(params[1].gpr()), GPRInfo::patchpointScratchRegister, ecmaMode,
2436                     node->op() == PutByIdDirect ? Direct : NotDirect);
2437
2438                 generator->generateFastPath(jit);
2439                 CCallHelpers::Label done = jit.label();
2440
2441                 params.addLatePath(
2442                     [=] (CCallHelpers& jit) {
2443                         AllowMacroScratchRegisterUsage allowScratch(jit);
2444
2445                         generator->slowPathJump().link(&jit);
2446                         CCallHelpers::Label slowPathBegin = jit.label();
2447                         CCallHelpers::Call slowPathCall = callOperation(
2448                             *state, params.unavailableRegisters(), jit, node->origin.semantic,
2449                             exceptions.get(), generator->slowPathFunction(), InvalidGPRReg,
2450                             CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
2451                             params[0].gpr(), CCallHelpers::TrustedImmPtr(uid)).call();
2452                         jit.jump().linkTo(done, &jit);
2453
2454                         generator->reportSlowPathCall(slowPathBegin, slowPathCall);
2455
2456                         jit.addLinkTask(
2457                             [=] (LinkBuffer& linkBuffer) {
2458                                 generator->finalize(linkBuffer);
2459                             });
2460                     });
2461             });
2462     }
2463
2464     void compileGetButterfly()
2465     {
2466         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2467     }
2468
2469     void compileConstantStoragePointer()
2470     {
2471         setStorage(m_out.constIntPtr(m_node->storagePointer()));
2472     }
2473
2474     void compileGetIndexedPropertyStorage()
2475     {
2476         LValue cell = lowCell(m_node->child1());
2477
2478         if (m_node->arrayMode().type() == Array::String) {
2479             LBasicBlock slowPath = m_out.newBlock();
2480             LBasicBlock continuation = m_out.newBlock();
2481
2482             LValue fastResultValue = m_out.loadPtr(cell, m_heaps.JSString_value);
2483             ValueFromBlock fastResult = m_out.anchor(fastResultValue);
2484
2485             m_out.branch(
2486                 m_out.notNull(fastResultValue), usually(continuation), rarely(slowPath));
2487
2488             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2489
2490             ValueFromBlock slowResult = m_out.anchor(
2491                 vmCall(m_out.intPtr, m_out.operation(operationResolveRope), m_callFrame, cell));
2492
2493             m_out.jump(continuation);
2494
2495             m_out.appendTo(continuation, lastNext);
2496
2497             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2498             return;
2499         }
2500
2501         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2502     }
2503
2504     void compileCheckArray()
2505     {
2506         Edge edge = m_node->child1();
2507         LValue cell = lowCell(edge);
2508
2509         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
2510             return;
2511
2512         speculate(
2513             BadIndexingType, jsValueValue(cell), 0,
2514             m_out.logicalNot(isArrayType(cell, m_node->arrayMode())));
2515     }
2516
2517     void compileGetTypedArrayByteOffset()
2518     {
2519         LValue basePtr = lowCell(m_node->child1());
2520
2521         LBasicBlock simpleCase = m_out.newBlock();
2522         LBasicBlock wastefulCase = m_out.newBlock();
2523         LBasicBlock continuation = m_out.newBlock();
2524
2525         LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2526         m_out.branch(
2527             m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2528             unsure(simpleCase), unsure(wastefulCase));
2529
2530         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2531
2532         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2533
2534         m_out.jump(continuation);
2535
2536         m_out.appendTo(wastefulCase, continuation);
2537
2538         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2539         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2540         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2541         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2542
2543         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2544
2545         m_out.jump(continuation);
2546         m_out.appendTo(continuation, lastNext);
2547
2548         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2549     }
2550
2551     void compileGetArrayLength()
2552     {
2553         switch (m_node->arrayMode().type()) {
2554         case Array::Int32:
2555         case Array::Double:
2556         case Array::Contiguous: {
2557             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2558             return;
2559         }
2560
2561         case Array::String: {
2562             LValue string = lowCell(m_node->child1());
2563             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2564             return;
2565         }
2566
2567         case Array::DirectArguments: {
2568             LValue arguments = lowCell(m_node->child1());
2569             speculate(
2570                 ExoticObjectMode, noValue(), nullptr,
2571                 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2572             setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2573             return;
2574         }
2575
2576         case Array::ScopedArguments: {
2577             LValue arguments = lowCell(m_node->child1());
2578             speculate(
2579                 ExoticObjectMode, noValue(), nullptr,
2580                 m_out.notZero32(m_out.load8ZeroExt32(arguments, m_heaps.ScopedArguments_overrodeThings)));
2581             setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2582             return;
2583         }
2584
2585         default:
2586             if (m_node->arrayMode().isSomeTypedArrayView()) {
2587                 setInt32(
2588                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2589                 return;
2590             }
2591
2592             DFG_CRASH(m_graph, m_node, "Bad array type");
2593             return;
2594         }
2595     }
2596
2597     void compileCheckInBounds()
2598     {
2599         speculate(
2600             OutOfBounds, noValue(), 0,
2601             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2602     }
2603
2604     void compileGetByVal()
2605     {
2606         switch (m_node->arrayMode().type()) {
2607         case Array::Int32:
2608         case Array::Contiguous: {
2609             LValue index = lowInt32(m_node->child2());
2610             LValue storage = lowStorage(m_node->child3());
2611
2612             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2613                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2614
2615             if (m_node->arrayMode().isInBounds()) {
2616                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2617                 LValue isHole = m_out.isZero64(result);
2618                 if (m_node->arrayMode().isSaneChain()) {
2619                     DFG_ASSERT(
2620                         m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous);
2621                     result = m_out.select(
2622                         isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
2623                 } else
2624                     speculate(LoadFromHole, noValue(), 0, isHole);
2625                 setJSValue(result);
2626                 return;
2627             }
2628
2629             LValue base = lowCell(m_node->child1());
2630
2631             LBasicBlock fastCase = m_out.newBlock();
2632             LBasicBlock slowCase = m_out.newBlock();
2633             LBasicBlock continuation = m_out.newBlock();
2634
2635             m_out.branch(
2636                 m_out.aboveOrEqual(
2637                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2638                 rarely(slowCase), usually(fastCase));
2639
2640             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2641
2642             LValue fastResultValue = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2643             ValueFromBlock fastResult = m_out.anchor(fastResultValue);
2644             m_out.branch(
2645                 m_out.isZero64(fastResultValue), rarely(slowCase), usually(continuation));
2646
2647             m_out.appendTo(slowCase, continuation);
2648             ValueFromBlock slowResult = m_out.anchor(
2649                 vmCall(m_out.int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2650             m_out.jump(continuation);
2651
2652             m_out.appendTo(continuation, lastNext);
2653             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2654             return;
2655         }
2656
2657         case Array::Double: {
2658             LValue index = lowInt32(m_node->child2());
2659             LValue storage = lowStorage(m_node->child3());
2660
2661             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2662
2663             if (m_node->arrayMode().isInBounds()) {
2664                 LValue result = m_out.loadDouble(
2665                     baseIndex(heap, storage, index, m_node->child2()));
2666
2667                 if (!m_node->arrayMode().isSaneChain()) {
2668                     speculate(
2669                         LoadFromHole, noValue(), 0,
2670                         m_out.doubleNotEqualOrUnordered(result, result));
2671                 }
2672                 setDouble(result);
2673                 break;
2674             }
2675
2676             LValue base = lowCell(m_node->child1());
2677
2678             LBasicBlock inBounds = m_out.newBlock();
2679             LBasicBlock boxPath = m_out.newBlock();
2680             LBasicBlock slowCase = m_out.newBlock();
2681             LBasicBlock continuation = m_out.newBlock();
2682
2683             m_out.branch(
2684                 m_out.aboveOrEqual(
2685                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2686                 rarely(slowCase), usually(inBounds));
2687
2688             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2689             LValue doubleValue = m_out.loadDouble(
2690                 baseIndex(heap, storage, index, m_node->child2()));
2691             m_out.branch(
2692                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2693                 rarely(slowCase), usually(boxPath));
2694
2695             m_out.appendTo(boxPath, slowCase);
2696             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2697             m_out.jump(continuation);
2698
2699             m_out.appendTo(slowCase, continuation);
2700             ValueFromBlock slowResult = m_out.anchor(
2701                 vmCall(m_out.int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2702             m_out.jump(continuation);
2703
2704             m_out.appendTo(continuation, lastNext);
2705             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2706             return;
2707         }
2708
2709         case Array::Undecided: {
2710             LValue index = lowInt32(m_node->child2());
2711
2712             speculate(OutOfBounds, noValue(), m_node, m_out.lessThan(index, m_out.int32Zero));
2713             setJSValue(m_out.constInt64(ValueUndefined));
2714             return;
2715         }
2716
2717         case Array::DirectArguments: {
2718             LValue base = lowCell(m_node->child1());
2719             LValue index = lowInt32(m_node->child2());
2720
2721             speculate(
2722                 ExoticObjectMode, noValue(), nullptr,
2723                 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2724             speculate(
2725                 ExoticObjectMode, noValue(), nullptr,
2726                 m_out.aboveOrEqual(
2727                     index,
2728                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2729
2730             TypedPointer address = m_out.baseIndex(
2731                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2732             setJSValue(m_out.load64(address));
2733             return;
2734         }
2735
2736         case Array::ScopedArguments: {
2737             LValue base = lowCell(m_node->child1());
2738             LValue index = lowInt32(m_node->child2());
2739
2740             speculate(
2741                 ExoticObjectMode, noValue(), nullptr,
2742                 m_out.aboveOrEqual(
2743                     index,
2744                     m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2745
2746             LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2747             LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2748
2749             LBasicBlock namedCase = m_out.newBlock();
2750             LBasicBlock overflowCase = m_out.newBlock();
2751             LBasicBlock continuation = m_out.newBlock();
2752
2753             m_out.branch(
2754                 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2755
2756             LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2757
2758             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2759             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2760
2761             TypedPointer address = m_out.baseIndex(
2762                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2763             LValue scopeOffset = m_out.load32(address);
2764
2765             speculate(
2766                 ExoticObjectMode, noValue(), nullptr,
2767                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2768
2769             address = m_out.baseIndex(
2770                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2771             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2772             m_out.jump(continuation);
2773
2774             m_out.appendTo(overflowCase, continuation);
2775
2776             address = m_out.baseIndex(
2777                 m_heaps.ScopedArguments_overflowStorage, base,
2778                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2779             LValue overflowValue = m_out.load64(address);
2780             speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2781             ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2782             m_out.jump(continuation);
2783
2784             m_out.appendTo(continuation, lastNext);
2785             setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2786             return;
2787         }
2788
2789         case Array::Generic: {
2790             setJSValue(vmCall(
2791                 m_out.int64, m_out.operation(operationGetByVal), m_callFrame,
2792                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2793             return;
2794         }
2795
2796         case Array::String: {
2797             compileStringCharAt();
2798             return;
2799         }
2800
2801         default: {
2802             LValue index = lowInt32(m_node->child2());
2803             LValue storage = lowStorage(m_node->child3());
2804
2805             TypedArrayType type = m_node->arrayMode().typedArrayType();
2806
2807             if (isTypedView(type)) {
2808                 TypedPointer pointer = TypedPointer(
2809                     m_heaps.typedArrayProperties,
2810                     m_out.add(
2811                         storage,
2812                         m_out.shl(
2813                             m_out.zeroExtPtr(index),
2814                             m_out.constIntPtr(logElementSize(type)))));
2815
2816                 if (isInt(type)) {
2817                     LValue result;
2818                     switch (elementSize(type)) {
2819                     case 1:
2820                         result = isSigned(type) ? m_out.load8SignExt32(pointer) :  m_out.load8ZeroExt32(pointer);
2821                         break;
2822                     case 2:
2823                         result = isSigned(type) ? m_out.load16SignExt32(pointer) :  m_out.load16ZeroExt32(pointer);
2824                         break;
2825                     case 4:
2826                         result = m_out.load32(pointer);
2827                         break;
2828                     default:
2829                         DFG_CRASH(m_graph, m_node, "Bad element size");
2830                     }
2831
2832                     if (elementSize(type) < 4 || isSigned(type)) {
2833                         setInt32(result);
2834                         return;
2835                     }
2836
2837                     if (m_node->shouldSpeculateInt32()) {
2838                         speculate(
2839                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2840                         setInt32(result);
2841                         return;
2842                     }
2843
2844                     if (m_node->shouldSpeculateMachineInt()) {
2845                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2846                         return;
2847                     }
2848
2849                     setDouble(m_out.unsignedToDouble(result));
2850                     return;
2851                 }
2852
2853                 ASSERT(isFloat(type));
2854
2855                 LValue result;
2856                 switch (type) {
2857                 case TypeFloat32:
2858                     result = m_out.floatToDouble(m_out.loadFloat(pointer));
2859                     break;
2860                 case TypeFloat64:
2861                     result = m_out.loadDouble(pointer);
2862                     break;
2863                 default:
2864                     DFG_CRASH(m_graph, m_node, "Bad typed array type");
2865                 }
2866
2867                 setDouble(result);
2868                 return;
2869             }
2870
2871             DFG_CRASH(m_graph, m_node, "Bad array type");
2872             return;
2873         } }
2874     }
2875
2876     void compileGetMyArgumentByVal()
2877     {
2878         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2879
2880         LValue index = lowInt32(m_node->child2());
2881
2882         LValue limit;
2883         if (inlineCallFrame && !inlineCallFrame->isVarargs())
2884             limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2885         else {
2886             VirtualRegister argumentCountRegister;
2887             if (!inlineCallFrame)
2888                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2889             else
2890                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2891             limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2892         }
2893
2894         speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2895
2896         TypedPointer base;
2897         if (inlineCallFrame) {
2898             if (inlineCallFrame->arguments.size() <= 1) {
2899                 // We should have already exited due to the bounds check, above. Just tell the
2900                 // compiler that anything dominated by this instruction is not reachable, so
2901                 // that we don't waste time generating such code. This will also plant some
2902                 // kind of crashing instruction so that if by some fluke the bounds check didn't
2903                 // work, we'll crash in an easy-to-see way.
2904                 didAlreadyTerminate();
2905                 return;
2906             }
2907             base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2908         } else
2909             base = addressFor(virtualRegisterForArgument(1));
2910
2911         LValue pointer = m_out.baseIndex(
2912             base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2913         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2914     }
2915
2916     void compilePutByVal()
2917     {
2918         Edge child1 = m_graph.varArgChild(m_node, 0);
2919         Edge child2 = m_graph.varArgChild(m_node, 1);
2920         Edge child3 = m_graph.varArgChild(m_node, 2);
2921         Edge child4 = m_graph.varArgChild(m_node, 3);
2922         Edge child5 = m_graph.varArgChild(m_node, 4);
2923
2924         switch (m_node->arrayMode().type()) {
2925         case Array::Generic: {
2926             V_JITOperation_EJJJ operation;
2927             if (m_node->op() == PutByValDirect) {
2928                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2929                     operation = operationPutByValDirectStrict;
2930                 else
2931                     operation = operationPutByValDirectNonStrict;
2932             } else {
2933                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2934                     operation = operationPutByValStrict;
2935                 else
2936                     operation = operationPutByValNonStrict;
2937             }
2938
2939             vmCall(
2940                 m_out.voidType, m_out.operation(operation), m_callFrame,
2941                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2942             return;
2943         }
2944
2945         default:
2946             break;
2947         }
2948
2949         LValue base = lowCell(child1);
2950         LValue index = lowInt32(child2);
2951         LValue storage = lowStorage(child4);
2952
2953         switch (m_node->arrayMode().type()) {
2954         case Array::Int32:
2955         case Array::Double:
2956         case Array::Contiguous: {
2957             LBasicBlock continuation = m_out.newBlock();
2958             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2959
2960             switch (m_node->arrayMode().type()) {
2961             case Array::Int32:
2962             case Array::Contiguous: {
2963                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2964
2965                 if (m_node->arrayMode().type() == Array::Int32)
2966                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2967
2968                 TypedPointer elementPointer = m_out.baseIndex(
2969                     m_node->arrayMode().type() == Array::Int32 ?
2970                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2971                     storage, m_out.zeroExtPtr(index), provenValue(child2));
2972
2973                 if (m_node->op() == PutByValAlias) {
2974                     m_out.store64(value, elementPointer);
2975                     break;
2976                 }
2977
2978                 contiguousPutByValOutOfBounds(
2979                     codeBlock()->isStrictMode()
2980                     ? operationPutByValBeyondArrayBoundsStrict
2981                     : operationPutByValBeyondArrayBoundsNonStrict,
2982                     base, storage, index, value, continuation);
2983
2984                 m_out.store64(value, elementPointer);
2985                 break;
2986             }
2987
2988             case Array::Double: {
2989                 LValue value = lowDouble(child3);
2990
2991                 FTL_TYPE_CHECK(
2992                     doubleValue(value), child3, SpecDoubleReal,
2993                     m_out.doubleNotEqualOrUnordered(value, value));
2994
2995                 TypedPointer elementPointer = m_out.baseIndex(
2996                     m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2997                     provenValue(child2));
2998
2999                 if (m_node->op() == PutByValAlias) {
3000                     m_out.storeDouble(value, elementPointer);
3001                     break;
3002                 }
3003
3004                 contiguousPutByValOutOfBounds(
3005                     codeBlock()->isStrictMode()
3006                     ? operationPutDoubleByValBeyondArrayBoundsStrict
3007                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
3008                     base, storage, index, value, continuation);
3009
3010                 m_out.storeDouble(value, elementPointer);
3011                 break;
3012             }
3013
3014             default:
3015                 DFG_CRASH(m_graph, m_node, "Bad array type");
3016             }
3017
3018             m_out.jump(continuation);
3019             m_out.appendTo(continuation, outerLastNext);
3020             return;
3021         }
3022
3023         default:
3024             TypedArrayType type = m_node->arrayMode().typedArrayType();
3025
3026             if (isTypedView(type)) {
3027                 TypedPointer pointer = TypedPointer(
3028                     m_heaps.typedArrayProperties,
3029                     m_out.add(
3030                         storage,
3031                         m_out.shl(
3032                             m_out.zeroExt(index, m_out.intPtr),
3033                             m_out.constIntPtr(logElementSize(type)))));
3034
3035                 Output::StoreType storeType;
3036                 LValue valueToStore;
3037
3038                 if (isInt(type)) {
3039                     LValue intValue;
3040                     switch (child3.useKind()) {
3041                     case Int52RepUse:
3042                     case Int32Use: {
3043                         if (child3.useKind() == Int32Use)
3044                             intValue = lowInt32(child3);
3045                         else
3046                             intValue = m_out.castToInt32(lowStrictInt52(child3));
3047
3048                         if (isClamped(type)) {
3049                             ASSERT(elementSize(type) == 1);
3050
3051                             LBasicBlock atLeastZero = m_out.newBlock();
3052                             LBasicBlock continuation = m_out.newBlock();
3053
3054                             Vector<ValueFromBlock, 2> intValues;
3055                             intValues.append(m_out.anchor(m_out.int32Zero));
3056                             m_out.branch(
3057                                 m_out.lessThan(intValue, m_out.int32Zero),
3058                                 unsure(continuation), unsure(atLeastZero));
3059
3060                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
3061
3062                             intValues.append(m_out.anchor(m_out.select(
3063                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
3064                                 m_out.constInt32(255),
3065                                 intValue)));
3066                             m_out.jump(continuation);
3067
3068                             m_out.appendTo(continuation, lastNext);
3069                             intValue = m_out.phi(m_out.int32, intValues);
3070                         }
3071                         break;
3072                     }
3073
3074                     case DoubleRepUse: {
3075                         LValue doubleValue = lowDouble(child3);
3076
3077                         if (isClamped(type)) {
3078                             ASSERT(elementSize(type) == 1);
3079
3080                             LBasicBlock atLeastZero = m_out.newBlock();
3081                             LBasicBlock withinRange = m_out.newBlock();
3082                             LBasicBlock continuation = m_out.newBlock();
3083
3084                             Vector<ValueFromBlock, 3> intValues;
3085                             intValues.append(m_out.anchor(m_out.int32Zero));
3086                             m_out.branch(
3087                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
3088                                 unsure(continuation), unsure(atLeastZero));
3089
3090                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
3091                             intValues.append(m_out.anchor(m_out.constInt32(255)));
3092                             m_out.branch(
3093                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
3094                                 unsure(continuation), unsure(withinRange));
3095
3096                             m_out.appendTo(withinRange, continuation);
3097                             intValues.append(m_out.anchor(m_out.doubleToInt(doubleValue)));
3098                             m_out.jump(continuation);
3099
3100                             m_out.appendTo(continuation, lastNext);
3101                             intValue = m_out.phi(m_out.int32, intValues);
3102                         } else
3103                             intValue = doubleToInt32(doubleValue);
3104                         break;
3105                     }
3106
3107                     default:
3108                         DFG_CRASH(m_graph, m_node, "Bad use kind");
3109                     }
3110
3111                     valueToStore = intValue;
3112                     switch (elementSize(type)) {
3113                     case 1:
3114                         storeType = Output::Store32As8;
3115                         break;
3116                     case 2:
3117                         storeType = Output::Store32As16;
3118                         break;
3119                     case 4:
3120                         storeType = Output::Store32;
3121                         break;
3122                     default:
3123                         DFG_CRASH(m_graph, m_node, "Bad element size");
3124                     }
3125                 } else /* !isInt(type) */ {
3126                     LValue value = lowDouble(child3);
3127                     switch (type) {
3128                     case TypeFloat32: