Remove MaximalFlushInsertionPhase

[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGByteCodeParser.cpp

/*
 * Copyright (C) 2011-2019 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "DFGByteCodeParser.h"

#if ENABLE(DFG_JIT)

#include "ArithProfile.h"
#include "ArrayConstructor.h"
#include "BasicBlockLocation.h"
#include "BuiltinNames.h"
#include "BytecodeStructs.h"
#include "CallLinkStatus.h"
#include "CodeBlock.h"
#include "CodeBlockWithJITType.h"
#include "CommonSlowPaths.h"
#include "DFGAbstractHeap.h"
#include "DFGArrayMode.h"
#include "DFGCFG.h"
#include "DFGCapabilities.h"
#include "DFGClobberize.h"
#include "DFGClobbersExitState.h"
#include "DFGGraph.h"
#include "DFGJITCode.h"
#include "FunctionCodeBlock.h"
#include "GetByIdStatus.h"
#include "Heap.h"
#include "InByIdStatus.h"
#include "InstanceOfStatus.h"
#include "JSCInlines.h"
#include "JSFixedArray.h"
#include "JSImmutableButterfly.h"
#include "JSModuleEnvironment.h"
#include "JSModuleNamespaceObject.h"
#include "NumberConstructor.h"
#include "ObjectConstructor.h"
#include "OpcodeInlines.h"
#include "PreciseJumpTargets.h"
#include "PutByIdFlags.h"
#include "PutByIdStatus.h"
#include "RegExpPrototype.h"
#include "StackAlignment.h"
#include "StringConstructor.h"
#include "StructureStubInfo.h"
#include "SymbolConstructor.h"
#include "Watchdog.h"
#include <wtf/CommaPrinter.h>
#include <wtf/HashMap.h>
#include <wtf/MathExtras.h>
#include <wtf/SetForScope.h>
#include <wtf/StdLibExtras.h>

namespace JSC { namespace DFG {

namespace DFGByteCodeParserInternal {
#ifdef NDEBUG
static const bool verbose = false;
#else
static const bool verbose = true;
#endif
} // namespace DFGByteCodeParserInternal

#define VERBOSE_LOG(...) do { \
if (DFGByteCodeParserInternal::verbose && Options::verboseDFGBytecodeParsing()) \
dataLog(__VA_ARGS__); \
} while (false)

// === ByteCodeParser ===
//
// This class is used to compile the dataflow graph from a CodeBlock.
class ByteCodeParser {
public:
    ByteCodeParser(Graph& graph)
        : m_vm(&graph.m_vm)
        , m_codeBlock(graph.m_codeBlock)
        , m_profiledBlock(graph.m_profiledBlock)
        , m_graph(graph)
        , m_currentBlock(0)
        , m_currentIndex(0)
        , m_constantUndefined(graph.freeze(jsUndefined()))
        , m_constantNull(graph.freeze(jsNull()))
        , m_constantNaN(graph.freeze(jsNumber(PNaN)))
        , m_constantOne(graph.freeze(jsNumber(1)))
        , m_numArguments(m_codeBlock->numParameters())
        , m_numLocals(m_codeBlock->numCalleeLocals())
        , m_parameterSlots(0)
        , m_numPassedVarArgs(0)
        , m_inlineStackTop(0)
        , m_currentInstruction(0)
        , m_hasDebuggerEnabled(graph.hasDebuggerEnabled())
    {
        ASSERT(m_profiledBlock);
    }
    
    // Parse a full CodeBlock of bytecode.
    void parse();
    
private:
    struct InlineStackEntry;

    // Just parse from m_currentIndex to the end of the current CodeBlock.
    void parseCodeBlock();
    
    void ensureLocals(unsigned newNumLocals)
    {
        VERBOSE_LOG("   ensureLocals: trying to raise m_numLocals from ", m_numLocals, " to ", newNumLocals, "\n");
        if (newNumLocals <= m_numLocals)
            return;
        m_numLocals = newNumLocals;
        for (size_t i = 0; i < m_graph.numBlocks(); ++i)
            m_graph.block(i)->ensureLocals(newNumLocals);
    }

    // Helper for min and max.
    template<typename ChecksFunctor>
    bool handleMinMax(VirtualRegister result, NodeType op, int registerOffset, int argumentCountIncludingThis, const ChecksFunctor& insertChecks);
    
    void refineStatically(CallLinkStatus&, Node* callTarget);
    // Blocks can either be targetable (i.e. in the m_blockLinkingTargets of one InlineStackEntry) with a well-defined bytecodeBegin,
    // or they can be untargetable, with bytecodeBegin==UINT_MAX, to be managed manually and not by the linkBlock machinery.
    // This is used most notably when doing polyvariant inlining (it requires a fair bit of control-flow with no bytecode analog).
    // It is also used when doing an early return from an inlined callee: it is easier to fix the bytecode index later on if needed
    // than to move the right index all the way to the treatment of op_ret.
    BasicBlock* allocateTargetableBlock(unsigned bytecodeIndex);
    BasicBlock* allocateUntargetableBlock();
    // An untargetable block can be given a bytecodeIndex to be later managed by linkBlock, but only once, and it can never go in the other direction
    void makeBlockTargetable(BasicBlock*, unsigned bytecodeIndex);
    void addJumpTo(BasicBlock*);
    void addJumpTo(unsigned bytecodeIndex);
    // Handle calls. This resolves issues surrounding inlining and intrinsics.
    enum Terminality { Terminal, NonTerminal };
    Terminality handleCall(
        VirtualRegister result, NodeType op, InlineCallFrame::Kind, unsigned instructionSize,
        Node* callTarget, int argumentCountIncludingThis, int registerOffset, CallLinkStatus,
        SpeculatedType prediction);
    template<typename CallOp>
    Terminality handleCall(const Instruction* pc, NodeType op, CallMode);
    template<typename CallOp>
    Terminality handleVarargsCall(const Instruction* pc, NodeType op, CallMode);
    void emitFunctionChecks(CallVariant, Node* callTarget, VirtualRegister thisArgumnt);
    void emitArgumentPhantoms(int registerOffset, int argumentCountIncludingThis);
    Node* getArgumentCount();
    template<typename ChecksFunctor>
    bool handleRecursiveTailCall(Node* callTargetNode, CallVariant, int registerOffset, int argumentCountIncludingThis, const ChecksFunctor& emitFunctionCheckIfNeeded);
    unsigned inliningCost(CallVariant, int argumentCountIncludingThis, InlineCallFrame::Kind); // Return UINT_MAX if it's not an inlining candidate. By convention, intrinsics have a cost of 1.
    // Handle inlining. Return true if it succeeded, false if we need to plant a call.
    bool handleVarargsInlining(Node* callTargetNode, VirtualRegister result, const CallLinkStatus&, int registerOffset, VirtualRegister thisArgument, VirtualRegister argumentsArgument, unsigned argumentsOffset, NodeType callOp, InlineCallFrame::Kind);
    unsigned getInliningBalance(const CallLinkStatus&, CodeSpecializationKind);
    enum class CallOptimizationResult { OptimizedToJump, Inlined, DidNothing };
    CallOptimizationResult handleCallVariant(Node* callTargetNode, VirtualRegister result, CallVariant, int registerOffset, VirtualRegister thisArgument, int argumentCountIncludingThis, unsigned nextOffset, InlineCallFrame::Kind, SpeculatedType prediction, unsigned& inliningBalance, BasicBlock* continuationBlock, bool needsToCheckCallee);
    CallOptimizationResult handleInlining(Node* callTargetNode, VirtualRegister result, const CallLinkStatus&, int registerOffset, VirtualRegister thisArgument, int argumentCountIncludingThis, unsigned nextOffset, NodeType callOp, InlineCallFrame::Kind, SpeculatedType prediction);
    template<typename ChecksFunctor>
    void inlineCall(Node* callTargetNode, VirtualRegister result, CallVariant, int registerOffset, int argumentCountIncludingThis, InlineCallFrame::Kind, BasicBlock* continuationBlock, const ChecksFunctor& insertChecks);
    // Handle intrinsic functions. Return true if it succeeded, false if we need to plant a call.
    template<typename ChecksFunctor>
    bool handleIntrinsicCall(Node* callee, VirtualRegister result, Intrinsic, int registerOffset, int argumentCountIncludingThis, SpeculatedType prediction, const ChecksFunctor& insertChecks);
    template<typename ChecksFunctor>
    bool handleDOMJITCall(Node* callee, VirtualRegister result, const DOMJIT::Signature*, int registerOffset, int argumentCountIncludingThis, SpeculatedType prediction, const ChecksFunctor& insertChecks);
    template<typename ChecksFunctor>
    bool handleIntrinsicGetter(VirtualRegister result, SpeculatedType prediction, const GetByIdVariant& intrinsicVariant, Node* thisNode, const ChecksFunctor& insertChecks);
    template<typename ChecksFunctor>
    bool handleTypedArrayConstructor(VirtualRegister result, InternalFunction*, int registerOffset, int argumentCountIncludingThis, TypedArrayType, const ChecksFunctor& insertChecks);
    template<typename ChecksFunctor>
    bool handleConstantInternalFunction(Node* callTargetNode, VirtualRegister result, InternalFunction*, int registerOffset, int argumentCountIncludingThis, CodeSpecializationKind, SpeculatedType, const ChecksFunctor& insertChecks);
    Node* handlePutByOffset(Node* base, unsigned identifier, PropertyOffset, Node* value);
    Node* handleGetByOffset(SpeculatedType, Node* base, unsigned identifierNumber, PropertyOffset, NodeType = GetByOffset);
    bool handleDOMJITGetter(VirtualRegister result, const GetByIdVariant&, Node* thisNode, unsigned identifierNumber, SpeculatedType prediction);
    bool handleModuleNamespaceLoad(VirtualRegister result, SpeculatedType, Node* base, GetByIdStatus);

    template<typename Bytecode>
    void handlePutByVal(Bytecode, unsigned instructionSize);
    template <typename Bytecode>
    void handlePutAccessorById(NodeType, Bytecode);
    template <typename Bytecode>
    void handlePutAccessorByVal(NodeType, Bytecode);
    template <typename Bytecode>
    void handleNewFunc(NodeType, Bytecode);
    template <typename Bytecode>
    void handleNewFuncExp(NodeType, Bytecode);

    // Create a presence ObjectPropertyCondition based on some known offset and structure set. Does not
    // check the validity of the condition, but it may return a null one if it encounters a contradiction.
    ObjectPropertyCondition presenceLike(
        JSObject* knownBase, UniquedStringImpl*, PropertyOffset, const StructureSet&);
    
    // Attempt to watch the presence of a property. It will watch that the property is present in the same
    // way as in all of the structures in the set. It may emit code instead of just setting a watchpoint.
    // Returns true if this all works out.
    bool checkPresenceLike(JSObject* knownBase, UniquedStringImpl*, PropertyOffset, const StructureSet&);
    void checkPresenceLike(Node* base, UniquedStringImpl*, PropertyOffset, const StructureSet&);
    
    // Works with both GetByIdVariant and the setter form of PutByIdVariant.
    template<typename VariantType>
    Node* load(SpeculatedType, Node* base, unsigned identifierNumber, const VariantType&);

    Node* store(Node* base, unsigned identifier, const PutByIdVariant&, Node* value);

    template<typename Op>
    void parseGetById(const Instruction*);
    void handleGetById(
        VirtualRegister destination, SpeculatedType, Node* base, unsigned identifierNumber, GetByIdStatus, AccessType, unsigned instructionSize);
    void emitPutById(
        Node* base, unsigned identifierNumber, Node* value,  const PutByIdStatus&, bool isDirect);
    void handlePutById(
        Node* base, unsigned identifierNumber, Node* value, const PutByIdStatus&,
        bool isDirect, unsigned intructionSize);
    
    // Either register a watchpoint or emit a check for this condition. Returns false if the
    // condition no longer holds, and therefore no reasonable check can be emitted.
    bool check(const ObjectPropertyCondition&);
    
    GetByOffsetMethod promoteToConstant(GetByOffsetMethod);
    
    // Either register a watchpoint or emit a check for this condition. It must be a Presence
    // condition. It will attempt to promote a Presence condition to an Equivalence condition.
    // Emits code for the loaded value that the condition guards, and returns a node containing
    // the loaded value. Returns null if the condition no longer holds.
    GetByOffsetMethod planLoad(const ObjectPropertyCondition&);
    Node* load(SpeculatedType, unsigned identifierNumber, const GetByOffsetMethod&, NodeType = GetByOffset);
    Node* load(SpeculatedType, const ObjectPropertyCondition&, NodeType = GetByOffset);
    
    // Calls check() for each condition in the set: that is, it either emits checks or registers
    // watchpoints (or a combination of the two) to make the conditions hold. If any of those
    // conditions are no longer checkable, returns false.
    bool check(const ObjectPropertyConditionSet&);
    
    // Calls check() for those conditions that aren't the slot base, and calls load() for the slot
    // base. Does a combination of watchpoint registration and check emission to guard the
    // conditions, and emits code to load the value from the slot base. Returns a node containing
    // the loaded value. Returns null if any of the conditions were no longer checkable.
    GetByOffsetMethod planLoad(const ObjectPropertyConditionSet&);
    Node* load(SpeculatedType, const ObjectPropertyConditionSet&, NodeType = GetByOffset);

    void prepareToParseBlock();
    void clearCaches();

    // Parse a single basic block of bytecode instructions.
    void parseBlock(unsigned limit);
    // Link block successors.
    void linkBlock(BasicBlock*, Vector<BasicBlock*>& possibleTargets);
    void linkBlocks(Vector<BasicBlock*>& unlinkedBlocks, Vector<BasicBlock*>& possibleTargets);
    
    VariableAccessData* newVariableAccessData(VirtualRegister operand)
    {
        ASSERT(!operand.isConstant());
        
        m_graph.m_variableAccessData.append(operand);
        return &m_graph.m_variableAccessData.last();
    }
    
    // Get/Set the operands/result of a bytecode instruction.
    Node* getDirect(VirtualRegister operand)
    {
        ASSERT(!operand.isConstant());

        // Is this an argument?
        if (operand.isArgument())
            return getArgument(operand);

        // Must be a local.
        return getLocal(operand);
    }

    Node* get(VirtualRegister operand)
    {
        if (operand.isConstant()) {
            unsigned constantIndex = operand.toConstantIndex();
            unsigned oldSize = m_constants.size();
            if (constantIndex >= oldSize || !m_constants[constantIndex]) {
                const CodeBlock& codeBlock = *m_inlineStackTop->m_codeBlock;
                JSValue value = codeBlock.getConstant(operand.offset());
                SourceCodeRepresentation sourceCodeRepresentation = codeBlock.constantSourceCodeRepresentation(operand.offset());
                if (constantIndex >= oldSize) {
                    m_constants.grow(constantIndex + 1);
                    for (unsigned i = oldSize; i < m_constants.size(); ++i)
                        m_constants[i] = nullptr;
                }

                Node* constantNode = nullptr;
                if (sourceCodeRepresentation == SourceCodeRepresentation::Double)
                    constantNode = addToGraph(DoubleConstant, OpInfo(m_graph.freezeStrong(jsDoubleNumber(value.asNumber()))));
                else
                    constantNode = addToGraph(JSConstant, OpInfo(m_graph.freezeStrong(value)));
                m_constants[constantIndex] = constantNode;
            }
            ASSERT(m_constants[constantIndex]);
            return m_constants[constantIndex];
        }
        
        if (inlineCallFrame()) {
            if (!inlineCallFrame()->isClosureCall) {
                JSFunction* callee = inlineCallFrame()->calleeConstant();
                if (operand.offset() == CallFrameSlot::callee)
                    return weakJSConstant(callee);
            }
        } else if (operand.offset() == CallFrameSlot::callee) {
            // We have to do some constant-folding here because this enables CreateThis folding. Note
            // that we don't have such watchpoint-based folding for inlined uses of Callee, since in that
            // case if the function is a singleton then we already know it.
            if (FunctionExecutable* executable = jsDynamicCast<FunctionExecutable*>(*m_vm, m_codeBlock->ownerExecutable())) {
                if (JSFunction* function = executable->singleton().inferredValue()) {
                    m_graph.watchpoints().addLazily(executable);
                    return weakJSConstant(function);
                }
            }
            return addToGraph(GetCallee);
        }
        
        return getDirect(m_inlineStackTop->remapOperand(operand));
    }
    
    enum SetMode {
        // A normal set which follows a two-phase commit that spans code origins. During
        // the current code origin it issues a MovHint, and at the start of the next
        // code origin there will be a SetLocal. If the local needs flushing, the second
        // SetLocal will be preceded with a Flush.
        NormalSet,
        
        // A set where the SetLocal happens immediately and there is still a Flush. This
        // is relevant when assigning to a local in tricky situations for the delayed
        // SetLocal logic but where we know that we have not performed any side effects
        // within this code origin. This is a safe replacement for NormalSet anytime we
        // know that we have not yet performed side effects in this code origin.
        ImmediateSetWithFlush,
        
        // A set where the SetLocal happens immediately and we do not Flush it even if
        // this is a local that is marked as needing it. This is relevant when
        // initializing locals at the top of a function.
        ImmediateNakedSet
    };
    Node* setDirect(VirtualRegister operand, Node* value, SetMode setMode = NormalSet)
    {
        addToGraph(MovHint, OpInfo(operand.offset()), value);

        // We can't exit anymore because our OSR exit state has changed.
        m_exitOK = false;

        DelayedSetLocal delayed(currentCodeOrigin(), operand, value, setMode);
        
        if (setMode == NormalSet) {
            m_setLocalQueue.append(delayed);
            return nullptr;
        }
        
        return delayed.execute(this);
    }
    
    void processSetLocalQueue()
    {
        for (unsigned i = 0; i < m_setLocalQueue.size(); ++i)
            m_setLocalQueue[i].execute(this);
        m_setLocalQueue.shrink(0);
    }

    Node* set(VirtualRegister operand, Node* value, SetMode setMode = NormalSet)
    {
        return setDirect(m_inlineStackTop->remapOperand(operand), value, setMode);
    }
    
    Node* injectLazyOperandSpeculation(Node* node)
    {
        ASSERT(node->op() == GetLocal);
        ASSERT(node->origin.semantic.bytecodeIndex() == m_currentIndex);
        ConcurrentJSLocker locker(m_inlineStackTop->m_profiledBlock->m_lock);
        LazyOperandValueProfileKey key(m_currentIndex, node->local());
        SpeculatedType prediction = m_inlineStackTop->m_lazyOperands.prediction(locker, key);
        node->variableAccessData()->predict(prediction);
        return node;
    }

    // Used in implementing get/set, above, where the operand is a local variable.
    Node* getLocal(VirtualRegister operand)
    {
        unsigned local = operand.toLocal();

        Node* node = m_currentBlock->variablesAtTail.local(local);
        
        // This has two goals: 1) link together variable access datas, and 2)
        // try to avoid creating redundant GetLocals. (1) is required for
        // correctness - no other phase will ensure that block-local variable
        // access data unification is done correctly. (2) is purely opportunistic
        // and is meant as an compile-time optimization only.
        
        VariableAccessData* variable;
        
        if (node) {
            variable = node->variableAccessData();
            
            switch (node->op()) {
            case GetLocal:
                return node;
            case SetLocal:
                return node->child1().node();
            default:
                break;
            }
        } else
            variable = newVariableAccessData(operand);
        
        node = injectLazyOperandSpeculation(addToGraph(GetLocal, OpInfo(variable)));
        m_currentBlock->variablesAtTail.local(local) = node;
        return node;
    }
    Node* setLocal(const CodeOrigin& semanticOrigin, VirtualRegister operand, Node* value, SetMode setMode = NormalSet)
    {
        SetForScope<CodeOrigin> originChange(m_currentSemanticOrigin, semanticOrigin);

        unsigned local = operand.toLocal();
        
        if (setMode != ImmediateNakedSet) {
            ArgumentPosition* argumentPosition = findArgumentPositionForLocal(operand);
            if (argumentPosition)
                flushDirect(operand, argumentPosition);
            else if (m_graph.needsScopeRegister() && operand == m_codeBlock->scopeRegister())
                flush(operand);
        }

        VariableAccessData* variableAccessData = newVariableAccessData(operand);
        variableAccessData->mergeStructureCheckHoistingFailed(
            m_inlineStackTop->m_exitProfile.hasExitSite(semanticOrigin.bytecodeIndex(), BadCache));
        variableAccessData->mergeCheckArrayHoistingFailed(
            m_inlineStackTop->m_exitProfile.hasExitSite(semanticOrigin.bytecodeIndex(), BadIndexingType));
        Node* node = addToGraph(SetLocal, OpInfo(variableAccessData), value);
        m_currentBlock->variablesAtTail.local(local) = node;
        return node;
    }

    // Used in implementing get/set, above, where the operand is an argument.
    Node* getArgument(VirtualRegister operand)
    {
        unsigned argument = operand.toArgument();
        ASSERT(argument < m_numArguments);
        
        Node* node = m_currentBlock->variablesAtTail.argument(argument);

        VariableAccessData* variable;
        
        if (node) {
            variable = node->variableAccessData();
            
            switch (node->op()) {
            case GetLocal:
                return node;
            case SetLocal:
                return node->child1().node();
            default:
                break;
            }
        } else
            variable = newVariableAccessData(operand);
        
        node = injectLazyOperandSpeculation(addToGraph(GetLocal, OpInfo(variable)));
        m_currentBlock->variablesAtTail.argument(argument) = node;
        return node;
    }
    Node* setArgument(const CodeOrigin& semanticOrigin, VirtualRegister operand, Node* value, SetMode setMode = NormalSet)
    {
        SetForScope<CodeOrigin> originChange(m_currentSemanticOrigin, semanticOrigin);

        unsigned argument = operand.toArgument();
        ASSERT(argument < m_numArguments);
        
        VariableAccessData* variableAccessData = newVariableAccessData(operand);

        // Always flush arguments, except for 'this'. If 'this' is created by us,
        // then make sure that it's never unboxed.
        if (argument || m_graph.needsFlushedThis()) {
            if (setMode != ImmediateNakedSet)
                flushDirect(operand);
        }
        
        if (!argument && m_codeBlock->specializationKind() == CodeForConstruct)
            variableAccessData->mergeShouldNeverUnbox(true);
        
        variableAccessData->mergeStructureCheckHoistingFailed(
            m_inlineStackTop->m_exitProfile.hasExitSite(semanticOrigin.bytecodeIndex(), BadCache));
        variableAccessData->mergeCheckArrayHoistingFailed(
            m_inlineStackTop->m_exitProfile.hasExitSite(semanticOrigin.bytecodeIndex(), BadIndexingType));
        Node* node = addToGraph(SetLocal, OpInfo(variableAccessData), value);
        m_currentBlock->variablesAtTail.argument(argument) = node;
        return node;
    }
    
    ArgumentPosition* findArgumentPositionForArgument(int argument)
    {
        InlineStackEntry* stack = m_inlineStackTop;
        while (stack->m_inlineCallFrame)
            stack = stack->m_caller;
        return stack->m_argumentPositions[argument];
    }
    
    ArgumentPosition* findArgumentPositionForLocal(VirtualRegister operand)
    {
        for (InlineStackEntry* stack = m_inlineStackTop; ; stack = stack->m_caller) {
            InlineCallFrame* inlineCallFrame = stack->m_inlineCallFrame;
            if (!inlineCallFrame)
                break;
            if (operand.offset() < static_cast<int>(inlineCallFrame->stackOffset + CallFrame::headerSizeInRegisters))
                continue;
            if (operand.offset() >= static_cast<int>(inlineCallFrame->stackOffset + CallFrame::thisArgumentOffset() + inlineCallFrame->argumentsWithFixup.size()))
                continue;
            int argument = VirtualRegister(operand.offset() - inlineCallFrame->stackOffset).toArgument();
            return stack->m_argumentPositions[argument];
        }
        return 0;
    }
    
    ArgumentPosition* findArgumentPosition(VirtualRegister operand)
    {
        if (operand.isArgument())
            return findArgumentPositionForArgument(operand.toArgument());
        return findArgumentPositionForLocal(operand);
    }

    template<typename AddFlushDirectFunc>
    void flushImpl(InlineCallFrame* inlineCallFrame, const AddFlushDirectFunc& addFlushDirect)
    {
        int numArguments;
        if (inlineCallFrame) {
            ASSERT(!m_graph.hasDebuggerEnabled());
            numArguments = inlineCallFrame->argumentsWithFixup.size();
            if (inlineCallFrame->isClosureCall)
                addFlushDirect(inlineCallFrame, remapOperand(inlineCallFrame, VirtualRegister(CallFrameSlot::callee)));
            if (inlineCallFrame->isVarargs())
                addFlushDirect(inlineCallFrame, remapOperand(inlineCallFrame, VirtualRegister(CallFrameSlot::argumentCount)));
        } else
            numArguments = m_graph.baselineCodeBlockFor(inlineCallFrame)->numParameters();

        for (unsigned argument = numArguments; argument--;)
            addFlushDirect(inlineCallFrame, remapOperand(inlineCallFrame, virtualRegisterForArgument(argument)));

        if (m_graph.needsScopeRegister())
            addFlushDirect(nullptr, m_graph.m_codeBlock->scopeRegister());
    }

    template<typename AddFlushDirectFunc, typename AddPhantomLocalDirectFunc>
    void flushForTerminalImpl(CodeOrigin origin, const AddFlushDirectFunc& addFlushDirect, const AddPhantomLocalDirectFunc& addPhantomLocalDirect)
    {
        origin.walkUpInlineStack(
            [&] (CodeOrigin origin) {
                unsigned bytecodeIndex = origin.bytecodeIndex();
                InlineCallFrame* inlineCallFrame = origin.inlineCallFrame();
                flushImpl(inlineCallFrame, addFlushDirect);

                CodeBlock* codeBlock = m_graph.baselineCodeBlockFor(inlineCallFrame);
                FullBytecodeLiveness& fullLiveness = m_graph.livenessFor(codeBlock);
                const FastBitVector& livenessAtBytecode = fullLiveness.getLiveness(bytecodeIndex);

                for (unsigned local = codeBlock->numCalleeLocals(); local--;) {
                    if (livenessAtBytecode[local])
                        addPhantomLocalDirect(inlineCallFrame, remapOperand(inlineCallFrame, virtualRegisterForLocal(local)));
                }
            });
    }

    void flush(VirtualRegister operand)
    {
        flushDirect(m_inlineStackTop->remapOperand(operand));
    }
    
    void flushDirect(VirtualRegister operand)
    {
        flushDirect(operand, findArgumentPosition(operand));
    }

    void flushDirect(VirtualRegister operand, ArgumentPosition* argumentPosition)
    {
        addFlushOrPhantomLocal<Flush>(operand, argumentPosition);
    }

    template<NodeType nodeType>
    void addFlushOrPhantomLocal(VirtualRegister operand, ArgumentPosition* argumentPosition)
    {
        ASSERT(!operand.isConstant());
        
        Node* node = m_currentBlock->variablesAtTail.operand(operand);
        
        VariableAccessData* variable;
        
        if (node)
            variable = node->variableAccessData();
        else
            variable = newVariableAccessData(operand);
        
        node = addToGraph(nodeType, OpInfo(variable));
        m_currentBlock->variablesAtTail.operand(operand) = node;
        if (argumentPosition)
            argumentPosition->addVariable(variable);
    }

    void phantomLocalDirect(VirtualRegister operand)
    {
        addFlushOrPhantomLocal<PhantomLocal>(operand, findArgumentPosition(operand));
    }

    void flush(InlineStackEntry* inlineStackEntry)
    {
        auto addFlushDirect = [&] (InlineCallFrame*, VirtualRegister reg) { flushDirect(reg); };
        flushImpl(inlineStackEntry->m_inlineCallFrame, addFlushDirect);
    }

    void flushForTerminal()
    {
        auto addFlushDirect = [&] (InlineCallFrame*, VirtualRegister reg) { flushDirect(reg); };
        auto addPhantomLocalDirect = [&] (InlineCallFrame*, VirtualRegister reg) { phantomLocalDirect(reg); };
        flushForTerminalImpl(currentCodeOrigin(), addFlushDirect, addPhantomLocalDirect);
    }

    void flushForReturn()
    {
        flush(m_inlineStackTop);
    }
    
    void flushIfTerminal(SwitchData& data)
    {
        if (data.fallThrough.bytecodeIndex() > m_currentIndex)
            return;
        
        for (unsigned i = data.cases.size(); i--;) {
            if (data.cases[i].target.bytecodeIndex() > m_currentIndex)
                return;
        }
        
        flushForTerminal();
    }

    // Assumes that the constant should be strongly marked.
    Node* jsConstant(JSValue constantValue)
    {
        return addToGraph(JSConstant, OpInfo(m_graph.freezeStrong(constantValue)));
    }

    Node* weakJSConstant(JSValue constantValue)
    {
        return addToGraph(JSConstant, OpInfo(m_graph.freeze(constantValue)));
    }

    // Helper functions to get/set the this value.
    Node* getThis()
    {
        return get(m_inlineStackTop->m_codeBlock->thisRegister());
    }

    void setThis(Node* value)
    {
        set(m_inlineStackTop->m_codeBlock->thisRegister(), value);
    }

    InlineCallFrame* inlineCallFrame()
    {
        return m_inlineStackTop->m_inlineCallFrame;
    }

    bool allInlineFramesAreTailCalls()
    {
        return !inlineCallFrame() || !inlineCallFrame()->getCallerSkippingTailCalls();
    }

    CodeOrigin currentCodeOrigin()
    {
        return CodeOrigin(m_currentIndex, inlineCallFrame());
    }

    NodeOrigin currentNodeOrigin()
    {
        CodeOrigin semantic;
        CodeOrigin forExit;

        if (m_currentSemanticOrigin.isSet())
            semantic = m_currentSemanticOrigin;
        else
            semantic = currentCodeOrigin();

        forExit = currentCodeOrigin();

        return NodeOrigin(semantic, forExit, m_exitOK);
    }
    
    BranchData* branchData(unsigned taken, unsigned notTaken)
    {
        // We assume that branches originating from bytecode always have a fall-through. We
        // use this assumption to avoid checking for the creation of terminal blocks.
        ASSERT((taken > m_currentIndex) || (notTaken > m_currentIndex));
        BranchData* data = m_graph.m_branchData.add();
        *data = BranchData::withBytecodeIndices(taken, notTaken);
        return data;
    }
    
    Node* addToGraph(Node* node)
    {
        VERBOSE_LOG("        appended ", node, " ", Graph::opName(node->op()), "\n");

        m_hasAnyForceOSRExits |= (node->op() == ForceOSRExit);

        m_currentBlock->append(node);
        if (clobbersExitState(m_graph, node))
            m_exitOK = false;
        return node;
    }
    
    Node* addToGraph(NodeType op, Node* child1 = 0, Node* child2 = 0, Node* child3 = 0)
    {
        Node* result = m_graph.addNode(
            op, currentNodeOrigin(), Edge(child1), Edge(child2),
            Edge(child3));
        return addToGraph(result);
    }
    Node* addToGraph(NodeType op, Edge child1, Edge child2 = Edge(), Edge child3 = Edge())
    {
        Node* result = m_graph.addNode(
            op, currentNodeOrigin(), child1, child2, child3);
        return addToGraph(result);
    }
    Node* addToGraph(NodeType op, OpInfo info, Node* child1 = 0, Node* child2 = 0, Node* child3 = 0)
    {
        Node* result = m_graph.addNode(
            op, currentNodeOrigin(), info, Edge(child1), Edge(child2),
            Edge(child3));
        return addToGraph(result);
    }
    Node* addToGraph(NodeType op, OpInfo info, Edge child1, Edge child2 = Edge(), Edge child3 = Edge())
    {
        Node* result = m_graph.addNode(op, currentNodeOrigin(), info, child1, child2, child3);
        return addToGraph(result);
    }
    Node* addToGraph(NodeType op, OpInfo info1, OpInfo info2, Node* child1 = 0, Node* child2 = 0, Node* child3 = 0)
    {
        Node* result = m_graph.addNode(
            op, currentNodeOrigin(), info1, info2,
            Edge(child1), Edge(child2), Edge(child3));
        return addToGraph(result);
    }
    Node* addToGraph(NodeType op, OpInfo info1, OpInfo info2, Edge child1, Edge child2 = Edge(), Edge child3 = Edge())
    {
        Node* result = m_graph.addNode(
            op, currentNodeOrigin(), info1, info2, child1, child2, child3);
        return addToGraph(result);
    }
    
    Node* addToGraph(Node::VarArgTag, NodeType op, OpInfo info1, OpInfo info2 = OpInfo())
    {
        Node* result = m_graph.addNode(
            Node::VarArg, op, currentNodeOrigin(), info1, info2,
            m_graph.m_varArgChildren.size() - m_numPassedVarArgs, m_numPassedVarArgs);
        addToGraph(result);
        
        m_numPassedVarArgs = 0;
        
        return result;
    }
    
    void addVarArgChild(Node* child)
    {
        m_graph.m_varArgChildren.append(Edge(child));
        m_numPassedVarArgs++;
    }

    void addVarArgChild(Edge child)
    {
        m_graph.m_varArgChildren.append(child);
        m_numPassedVarArgs++;
    }
    
    Node* addCallWithoutSettingResult(
        NodeType op, OpInfo opInfo, Node* callee, int argCount, int registerOffset,
        OpInfo prediction)
    {
        addVarArgChild(callee);
        size_t parameterSlots = Graph::parameterSlotsForArgCount(argCount);

        if (parameterSlots > m_parameterSlots)
            m_parameterSlots = parameterSlots;

        for (int i = 0; i < argCount; ++i)
            addVarArgChild(get(virtualRegisterForArgument(i, registerOffset)));

        return addToGraph(Node::VarArg, op, opInfo, prediction);
    }
    
    Node* addCall(
        VirtualRegister result, NodeType op, const DOMJIT::Signature* signature, Node* callee, int argCount, int registerOffset,
        SpeculatedType prediction)
    {
        if (op == TailCall) {
            if (allInlineFramesAreTailCalls())
                return addCallWithoutSettingResult(op, OpInfo(signature), callee, argCount, registerOffset, OpInfo());
            op = TailCallInlinedCaller;
        }


        Node* call = addCallWithoutSettingResult(
            op, OpInfo(signature), callee, argCount, registerOffset, OpInfo(prediction));
        if (result.isValid())
            set(result, call);
        return call;
    }
    
    Node* cellConstantWithStructureCheck(JSCell* object, Structure* structure)
    {
        // FIXME: This should route to emitPropertyCheck, not the other way around. But currently,
        // this gets no profit from using emitPropertyCheck() since we'll non-adaptively watch the
        // object's structure as soon as we make it a weakJSCosntant.
        Node* objectNode = weakJSConstant(object);
        addToGraph(CheckStructure, OpInfo(m_graph.addStructureSet(structure)), objectNode);
        return objectNode;
    }
    
    SpeculatedType getPredictionWithoutOSRExit(unsigned bytecodeIndex)
    {
        auto getValueProfilePredictionFromForCodeBlockAndBytecodeOffset = [&] (CodeBlock* codeBlock, const CodeOrigin& codeOrigin)
        {
            SpeculatedType prediction;
            {
                ConcurrentJSLocker locker(codeBlock->m_lock);
                prediction = codeBlock->valueProfilePredictionForBytecodeOffset(locker, codeOrigin.bytecodeIndex());
            }
            auto* fuzzerAgent = m_vm->fuzzerAgent();
            if (UNLIKELY(fuzzerAgent))
                return fuzzerAgent->getPrediction(codeBlock, codeOrigin, prediction) & SpecBytecodeTop;
            return prediction;
        };

        SpeculatedType prediction = getValueProfilePredictionFromForCodeBlockAndBytecodeOffset(m_inlineStackTop->m_profiledBlock, CodeOrigin(bytecodeIndex, inlineCallFrame()));
        if (prediction != SpecNone)
            return prediction;

        // If we have no information about the values this
        // node generates, we check if by any chance it is
        // a tail call opcode. In that case, we walk up the
        // inline frames to find a call higher in the call
        // chain and use its prediction. If we only have
        // inlined tail call frames, we use SpecFullTop
        // to avoid a spurious OSR exit.
        auto instruction = m_inlineStackTop->m_profiledBlock->instructions().at(bytecodeIndex);
        OpcodeID opcodeID = instruction->opcodeID();

        switch (opcodeID) {
        case op_tail_call:
        case op_tail_call_varargs:
        case op_tail_call_forward_arguments: {
            // Things should be more permissive to us returning BOTTOM instead of TOP here.
            // Currently, this will cause us to Force OSR exit. This is bad because returning
            // TOP will cause anything that transitively touches this speculated type to
            // also become TOP during prediction propagation.
            // https://bugs.webkit.org/show_bug.cgi?id=164337
            if (!inlineCallFrame())
                return SpecFullTop;

            CodeOrigin* codeOrigin = inlineCallFrame()->getCallerSkippingTailCalls();
            if (!codeOrigin)
                return SpecFullTop;

            InlineStackEntry* stack = m_inlineStackTop;
            while (stack->m_inlineCallFrame != codeOrigin->inlineCallFrame())
                stack = stack->m_caller;

            return getValueProfilePredictionFromForCodeBlockAndBytecodeOffset(stack->m_profiledBlock, *codeOrigin);
        }

        default:
            return SpecNone;
        }

        RELEASE_ASSERT_NOT_REACHED();
        return SpecNone;
    }

    SpeculatedType getPrediction(unsigned bytecodeIndex)
    {
        SpeculatedType prediction = getPredictionWithoutOSRExit(bytecodeIndex);

        if (prediction == SpecNone) {
            // We have no information about what values this node generates. Give up
            // on executing this code, since we're likely to do more damage than good.
            addToGraph(ForceOSRExit);
        }
        
        return prediction;
    }
    
    SpeculatedType getPredictionWithoutOSRExit()
    {
        return getPredictionWithoutOSRExit(m_currentIndex);
    }
    
    SpeculatedType getPrediction()
    {
        return getPrediction(m_currentIndex);
    }
    
    ArrayMode getArrayMode(Array::Action action)
    {
        CodeBlock* codeBlock = m_inlineStackTop->m_profiledBlock;
        ArrayProfile* profile = codeBlock->getArrayProfile(codeBlock->bytecodeOffset(m_currentInstruction));
        return getArrayMode(*profile, action);
    }

    ArrayMode getArrayMode(ArrayProfile& profile, Array::Action action)
    {
        ConcurrentJSLocker locker(m_inlineStackTop->m_profiledBlock->m_lock);
        profile.computeUpdatedPrediction(locker, m_inlineStackTop->m_profiledBlock);
        bool makeSafe = profile.outOfBounds(locker);
        return ArrayMode::fromObserved(locker, &profile, action, makeSafe);
    }

    Node* makeSafe(Node* node)
    {
        if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, Overflow))
            node->mergeFlags(NodeMayOverflowInt32InDFG);
        if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, NegativeZero))
            node->mergeFlags(NodeMayNegZeroInDFG);
        
        if (!isX86() && (node->op() == ArithMod || node->op() == ValueMod))
            return node;

        {
            ArithProfile* arithProfile = m_inlineStackTop->m_profiledBlock->arithProfileForBytecodeOffset(m_currentIndex);
            if (arithProfile) {
                switch (node->op()) {
                case ArithAdd:
                case ArithSub:
                case ValueAdd:
                    if (arithProfile->didObserveDouble())
                        node->mergeFlags(NodeMayHaveDoubleResult);
                    if (arithProfile->didObserveNonNumeric())
                        node->mergeFlags(NodeMayHaveNonNumericResult);
                    if (arithProfile->didObserveBigInt())
                        node->mergeFlags(NodeMayHaveBigIntResult);
                    break;
                
                case ValueMul:
                case ArithMul: {
                    if (arithProfile->didObserveInt52Overflow())
                        node->mergeFlags(NodeMayOverflowInt52);
                    if (arithProfile->didObserveInt32Overflow() || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, Overflow))
                        node->mergeFlags(NodeMayOverflowInt32InBaseline);
                    if (arithProfile->didObserveNegZeroDouble() || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, NegativeZero))
                        node->mergeFlags(NodeMayNegZeroInBaseline);
                    if (arithProfile->didObserveDouble())
                        node->mergeFlags(NodeMayHaveDoubleResult);
                    if (arithProfile->didObserveNonNumeric())
                        node->mergeFlags(NodeMayHaveNonNumericResult);
                    if (arithProfile->didObserveBigInt())
                        node->mergeFlags(NodeMayHaveBigIntResult);
                    break;
                }
                case ValueNegate:
                case ArithNegate: {
                    if (arithProfile->lhsObservedType().sawNumber() || arithProfile->didObserveDouble())
                        node->mergeFlags(NodeMayHaveDoubleResult);
                    if (arithProfile->didObserveNegZeroDouble() || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, NegativeZero))
                        node->mergeFlags(NodeMayNegZeroInBaseline);
                    if (arithProfile->didObserveInt32Overflow() || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, Overflow))
                        node->mergeFlags(NodeMayOverflowInt32InBaseline);
                    if (arithProfile->didObserveNonNumeric())
                        node->mergeFlags(NodeMayHaveNonNumericResult);
                    if (arithProfile->didObserveBigInt())
                        node->mergeFlags(NodeMayHaveBigIntResult);
                    break;
                }
                
                default:
                    break;
                }
            }
        }
        
        if (m_inlineStackTop->m_profiledBlock->likelyToTakeSlowCase(m_currentIndex)) {
            switch (node->op()) {
            case UInt32ToNumber:
            case ArithAdd:
            case ArithSub:
            case ValueAdd:
            case ValueMod:
            case ArithMod: // for ArithMod "MayOverflow" means we tried to divide by zero, or we saw double.
                node->mergeFlags(NodeMayOverflowInt32InBaseline);
                break;
                
            default:
                break;
            }
        }
        
        return node;
    }
    
    Node* makeDivSafe(Node* node)
    {
        ASSERT(node->op() == ArithDiv || node->op() == ValueDiv);
        
        if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, Overflow))
            node->mergeFlags(NodeMayOverflowInt32InDFG);
        if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, NegativeZero))
            node->mergeFlags(NodeMayNegZeroInDFG);
        
        // The main slow case counter for op_div in the old JIT counts only when
        // the operands are not numbers. We don't care about that since we already
        // have speculations in place that take care of that separately. We only
        // care about when the outcome of the division is not an integer, which
        // is what the special fast case counter tells us.
        
        if (!m_inlineStackTop->m_profiledBlock->couldTakeSpecialFastCase(m_currentIndex))
            return node;
        
        // FIXME: It might be possible to make this more granular.
        node->mergeFlags(NodeMayOverflowInt32InBaseline | NodeMayNegZeroInBaseline);
        
        ArithProfile* arithProfile = m_inlineStackTop->m_profiledBlock->arithProfileForBytecodeOffset(m_currentIndex);
        if (arithProfile->didObserveBigInt())
            node->mergeFlags(NodeMayHaveBigIntResult);

        return node;
    }
    
    void noticeArgumentsUse()
    {
        // All of the arguments in this function need to be formatted as JSValues because we will
        // load from them in a random-access fashion and we don't want to have to switch on
        // format.
        
        for (ArgumentPosition* argument : m_inlineStackTop->m_argumentPositions)
            argument->mergeShouldNeverUnbox(true);
    }

    bool needsDynamicLookup(ResolveType, OpcodeID);

    VM* m_vm;
    CodeBlock* m_codeBlock;
    CodeBlock* m_profiledBlock;
    Graph& m_graph;

    // The current block being generated.
    BasicBlock* m_currentBlock;
    // The bytecode index of the current instruction being generated.
    unsigned m_currentIndex;
    // The semantic origin of the current node if different from the current Index.
    CodeOrigin m_currentSemanticOrigin;
    // True if it's OK to OSR exit right now.
    bool m_exitOK { false };

    FrozenValue* m_constantUndefined;
    FrozenValue* m_constantNull;
    FrozenValue* m_constantNaN;
    FrozenValue* m_constantOne;
    Vector<Node*, 16> m_constants;

    HashMap<InlineCallFrame*, Vector<ArgumentPosition*>, WTF::DefaultHash<InlineCallFrame*>::Hash, WTF::NullableHashTraits<InlineCallFrame*>> m_inlineCallFrameToArgumentPositions;

    // The number of arguments passed to the function.
    unsigned m_numArguments;
    // The number of locals (vars + temporaries) used in the function.
    unsigned m_numLocals;
    // The number of slots (in units of sizeof(Register)) that we need to
    // preallocate for arguments to outgoing calls from this frame. This
    // number includes the CallFrame slots that we initialize for the callee
    // (but not the callee-initialized CallerFrame and ReturnPC slots).
    // This number is 0 if and only if this function is a leaf.
    unsigned m_parameterSlots;
    // The number of var args passed to the next var arg node.
    unsigned m_numPassedVarArgs;

    struct InlineStackEntry {
        ByteCodeParser* m_byteCodeParser;
        
        CodeBlock* m_codeBlock;
        CodeBlock* m_profiledBlock;
        InlineCallFrame* m_inlineCallFrame;
        
        ScriptExecutable* executable() { return m_codeBlock->ownerExecutable(); }
        
        QueryableExitProfile m_exitProfile;
        
        // Remapping of identifier and constant numbers from the code block being
        // inlined (inline callee) to the code block that we're inlining into
        // (the machine code block, which is the transitive, though not necessarily
        // direct, caller).
        Vector<unsigned> m_identifierRemap;
        Vector<unsigned> m_switchRemap;
        
        // These are blocks whose terminal is a Jump, Branch or Switch, and whose target has not yet been linked.
        // Their terminal instead refers to a bytecode index, and the right BB can be found in m_blockLinkingTargets.
        Vector<BasicBlock*> m_unlinkedBlocks;
        
        // Potential block linking targets. Must be sorted by bytecodeBegin, and
        // cannot have two blocks that have the same bytecodeBegin.
        Vector<BasicBlock*> m_blockLinkingTargets;

        // Optional: a continuation block for returns to jump to. It is set by early returns if it does not exist.
        BasicBlock* m_continuationBlock;

        VirtualRegister m_returnValue;
        
        // Speculations about variable types collected from the profiled code block,
        // which are based on OSR exit profiles that past DFG compilations of this
        // code block had gathered.
        LazyOperandValueProfileParser m_lazyOperands;
        
        ICStatusMap m_baselineMap;
        ICStatusContext m_optimizedContext;
        
        // Pointers to the argument position trackers for this slice of code.
        Vector<ArgumentPosition*> m_argumentPositions;
        
        InlineStackEntry* m_caller;
        
        InlineStackEntry(
            ByteCodeParser*,
            CodeBlock*,
            CodeBlock* profiledBlock,
            JSFunction* callee, // Null if this is a closure call.
            VirtualRegister returnValueVR,
            VirtualRegister inlineCallFrameStart,
            int argumentCountIncludingThis,
            InlineCallFrame::Kind,
            BasicBlock* continuationBlock);
        
        ~InlineStackEntry();
        
        VirtualRegister remapOperand(VirtualRegister operand) const
        {
            if (!m_inlineCallFrame)
                return operand;
            
            ASSERT(!operand.isConstant());

            return VirtualRegister(operand.offset() + m_inlineCallFrame->stackOffset);
        }
    };
    
    InlineStackEntry* m_inlineStackTop;
    
    ICStatusContextStack m_icContextStack;
    
    struct DelayedSetLocal {
        CodeOrigin m_origin;
        VirtualRegister m_operand;
        Node* m_value;
        SetMode m_setMode;
        
        DelayedSetLocal() { }
        DelayedSetLocal(const CodeOrigin& origin, VirtualRegister operand, Node* value, SetMode setMode)
            : m_origin(origin)
            , m_operand(operand)
            , m_value(value)
            , m_setMode(setMode)
        {
            RELEASE_ASSERT(operand.isValid());
        }
        
        Node* execute(ByteCodeParser* parser)
        {
            if (m_operand.isArgument())
                return parser->setArgument(m_origin, m_operand, m_value, m_setMode);
            return parser->setLocal(m_origin, m_operand, m_value, m_setMode);
        }
    };
    
    Vector<DelayedSetLocal, 2> m_setLocalQueue;

    const Instruction* m_currentInstruction;
    bool m_hasDebuggerEnabled;
    bool m_hasAnyForceOSRExits { false };
};

BasicBlock* ByteCodeParser::allocateTargetableBlock(unsigned bytecodeIndex)
{
    ASSERT(bytecodeIndex != UINT_MAX);
    Ref<BasicBlock> block = adoptRef(*new BasicBlock(bytecodeIndex, m_numArguments, m_numLocals, 1));
    BasicBlock* blockPtr = block.ptr();
    // m_blockLinkingTargets must always be sorted in increasing order of bytecodeBegin
    if (m_inlineStackTop->m_blockLinkingTargets.size())
        ASSERT(m_inlineStackTop->m_blockLinkingTargets.last()->bytecodeBegin < bytecodeIndex);
    m_inlineStackTop->m_blockLinkingTargets.append(blockPtr);
    m_graph.appendBlock(WTFMove(block));
    return blockPtr;
}

BasicBlock* ByteCodeParser::allocateUntargetableBlock()
{
    Ref<BasicBlock> block = adoptRef(*new BasicBlock(UINT_MAX, m_numArguments, m_numLocals, 1));
    BasicBlock* blockPtr = block.ptr();
    m_graph.appendBlock(WTFMove(block));
    return blockPtr;
}

void ByteCodeParser::makeBlockTargetable(BasicBlock* block, unsigned bytecodeIndex)
{
    RELEASE_ASSERT(block->bytecodeBegin == UINT_MAX);
    block->bytecodeBegin = bytecodeIndex;
    // m_blockLinkingTargets must always be sorted in increasing order of bytecodeBegin
    if (m_inlineStackTop->m_blockLinkingTargets.size())
        ASSERT(m_inlineStackTop->m_blockLinkingTargets.last()->bytecodeBegin < bytecodeIndex);
    m_inlineStackTop->m_blockLinkingTargets.append(block);
}

void ByteCodeParser::addJumpTo(BasicBlock* block)
{
    ASSERT(!m_currentBlock->terminal());
    Node* jumpNode = addToGraph(Jump);
    jumpNode->targetBlock() = block;
    m_currentBlock->didLink();
}

void ByteCodeParser::addJumpTo(unsigned bytecodeIndex)
{
    ASSERT(!m_currentBlock->terminal());
    addToGraph(Jump, OpInfo(bytecodeIndex));
    m_inlineStackTop->m_unlinkedBlocks.append(m_currentBlock);
}

template<typename CallOp>
ByteCodeParser::Terminality ByteCodeParser::handleCall(const Instruction* pc, NodeType op, CallMode callMode)
{
    auto bytecode = pc->as<CallOp>();
    Node* callTarget = get(bytecode.m_callee);
    int registerOffset = -static_cast<int>(bytecode.m_argv);

    CallLinkStatus callLinkStatus = CallLinkStatus::computeFor(
        m_inlineStackTop->m_profiledBlock, currentCodeOrigin(),
        m_inlineStackTop->m_baselineMap, m_icContextStack);

    InlineCallFrame::Kind kind = InlineCallFrame::kindFor(callMode);

    return handleCall(bytecode.m_dst, op, kind, pc->size(), callTarget,
        bytecode.m_argc, registerOffset, callLinkStatus, getPrediction());
}

void ByteCodeParser::refineStatically(CallLinkStatus& callLinkStatus, Node* callTarget)
{
    if (callTarget->isCellConstant())
        callLinkStatus.setProvenConstantCallee(CallVariant(callTarget->asCell()));
}

ByteCodeParser::Terminality ByteCodeParser::handleCall(
    VirtualRegister result, NodeType op, InlineCallFrame::Kind kind, unsigned instructionSize,
    Node* callTarget, int argumentCountIncludingThis, int registerOffset,
    CallLinkStatus callLinkStatus, SpeculatedType prediction)
{
    ASSERT(registerOffset <= 0);

    refineStatically(callLinkStatus, callTarget);
    
    VERBOSE_LOG("    Handling call at ", currentCodeOrigin(), ": ", callLinkStatus, "\n");
    
    // If we have profiling information about this call, and it did not behave too polymorphically,
    // we may be able to inline it, or in the case of recursive tail calls turn it into a jump.
    if (callLinkStatus.canOptimize()) {
        addToGraph(FilterCallLinkStatus, OpInfo(m_graph.m_plan.recordedStatuses().addCallLinkStatus(currentCodeOrigin(), callLinkStatus)), callTarget);

        VirtualRegister thisArgument = virtualRegisterForArgument(0, registerOffset);
        auto optimizationResult = handleInlining(callTarget, result, callLinkStatus, registerOffset, thisArgument,
            argumentCountIncludingThis, m_currentIndex + instructionSize, op, kind, prediction);
        if (optimizationResult == CallOptimizationResult::OptimizedToJump)
            return Terminal;
        if (optimizationResult == CallOptimizationResult::Inlined) {
            if (UNLIKELY(m_graph.compilation()))
                m_graph.compilation()->noticeInlinedCall();
            return NonTerminal;
        }
    }
    
    Node* callNode = addCall(result, op, nullptr, callTarget, argumentCountIncludingThis, registerOffset, prediction);
    ASSERT(callNode->op() != TailCallVarargs && callNode->op() != TailCallForwardVarargs);
    return callNode->op() == TailCall ? Terminal : NonTerminal;
}

template<typename CallOp>
ByteCodeParser::Terminality ByteCodeParser::handleVarargsCall(const Instruction* pc, NodeType op, CallMode callMode)
{
    auto bytecode = pc->as<CallOp>();
    int firstFreeReg = bytecode.m_firstFree.offset();
    int firstVarArgOffset = bytecode.m_firstVarArg;
    
    SpeculatedType prediction = getPrediction();
    
    Node* callTarget = get(bytecode.m_callee);
    
    CallLinkStatus callLinkStatus = CallLinkStatus::computeFor(
        m_inlineStackTop->m_profiledBlock, currentCodeOrigin(),
        m_inlineStackTop->m_baselineMap, m_icContextStack);
    refineStatically(callLinkStatus, callTarget);
    
    VERBOSE_LOG("    Varargs call link status at ", currentCodeOrigin(), ": ", callLinkStatus, "\n");
    
    if (callLinkStatus.canOptimize()) {
        addToGraph(FilterCallLinkStatus, OpInfo(m_graph.m_plan.recordedStatuses().addCallLinkStatus(currentCodeOrigin(), callLinkStatus)), callTarget);

        if (handleVarargsInlining(callTarget, bytecode.m_dst,
            callLinkStatus, firstFreeReg, bytecode.m_thisValue, bytecode.m_arguments,
            firstVarArgOffset, op,
            InlineCallFrame::varargsKindFor(callMode))) {
            if (UNLIKELY(m_graph.compilation()))
                m_graph.compilation()->noticeInlinedCall();
            return NonTerminal;
        }
    }
    
    CallVarargsData* data = m_graph.m_callVarargsData.add();
    data->firstVarArgOffset = firstVarArgOffset;
    
    Node* thisChild = get(bytecode.m_thisValue);
    Node* argumentsChild = nullptr;
    if (op != TailCallForwardVarargs)
        argumentsChild = get(bytecode.m_arguments);

    if (op == TailCallVarargs || op == TailCallForwardVarargs) {
        if (allInlineFramesAreTailCalls()) {
            addToGraph(op, OpInfo(data), OpInfo(), callTarget, thisChild, argumentsChild);
            return Terminal;
        }
        op = op == TailCallVarargs ? TailCallVarargsInlinedCaller : TailCallForwardVarargsInlinedCaller;
    }

    Node* call = addToGraph(op, OpInfo(data), OpInfo(prediction), callTarget, thisChild, argumentsChild);
    if (bytecode.m_dst.isValid())
        set(bytecode.m_dst, call);
    return NonTerminal;
}

void ByteCodeParser::emitFunctionChecks(CallVariant callee, Node* callTarget, VirtualRegister thisArgumentReg)
{
    Node* thisArgument;
    if (thisArgumentReg.isValid())
        thisArgument = get(thisArgumentReg);
    else
        thisArgument = nullptr;

    JSCell* calleeCell;
    Node* callTargetForCheck;
    if (callee.isClosureCall()) {
        calleeCell = callee.executable();
        callTargetForCheck = addToGraph(GetExecutable, callTarget);
    } else {
        calleeCell = callee.nonExecutableCallee();
        callTargetForCheck = callTarget;
    }
    
    ASSERT(calleeCell);
    addToGraph(CheckCell, OpInfo(m_graph.freeze(calleeCell)), callTargetForCheck);
    if (thisArgument)
        addToGraph(Phantom, thisArgument);
}

Node* ByteCodeParser::getArgumentCount()
{
    Node* argumentCount;
    if (m_inlineStackTop->m_inlineCallFrame && !m_inlineStackTop->m_inlineCallFrame->isVarargs())
        argumentCount = jsConstant(m_graph.freeze(jsNumber(m_inlineStackTop->m_inlineCallFrame->argumentCountIncludingThis))->value());
    else
        argumentCount = addToGraph(GetArgumentCountIncludingThis, OpInfo(m_inlineStackTop->m_inlineCallFrame), OpInfo(SpecInt32Only));
    return argumentCount;
}

void ByteCodeParser::emitArgumentPhantoms(int registerOffset, int argumentCountIncludingThis)
{
    for (int i = 0; i < argumentCountIncludingThis; ++i)
        addToGraph(Phantom, get(virtualRegisterForArgument(i, registerOffset)));
}

template<typename ChecksFunctor>
bool ByteCodeParser::handleRecursiveTailCall(Node* callTargetNode, CallVariant callVariant, int registerOffset, int argumentCountIncludingThis, const ChecksFunctor& emitFunctionCheckIfNeeded)
{
    if (UNLIKELY(!Options::optimizeRecursiveTailCalls()))
        return false;

    auto targetExecutable = callVariant.executable();
    InlineStackEntry* stackEntry = m_inlineStackTop;
    do {
        if (targetExecutable != stackEntry->executable())
            continue;
        VERBOSE_LOG("   We found a recursive tail call, trying to optimize it into a jump.\n");

        if (auto* callFrame = stackEntry->m_inlineCallFrame) {
            // Some code may statically use the argument count from the InlineCallFrame, so it would be invalid to loop back if it does not match.
            // We "continue" instead of returning false in case another stack entry further on the stack has the right number of arguments.
            if (argumentCountIncludingThis != static_cast<int>(callFrame->argumentCountIncludingThis))
                continue;
        } else {
            // We are in the machine code entry (i.e. the original caller).
            // If we have more arguments than the number of parameters to the function, it is not clear where we could put them on the stack.
            if (argumentCountIncludingThis > m_codeBlock->numParameters())
                return false;
        }

        // If an InlineCallFrame is not a closure, it was optimized using a constant callee.
        // Check if this is the same callee that we try to inline here.
        if (stackEntry->m_inlineCallFrame && !stackEntry->m_inlineCallFrame->isClosureCall) {
            if (stackEntry->m_inlineCallFrame->calleeConstant() != callVariant.function())
                continue;
        }

        // We must add some check that the profiling information was correct and the target of this call is what we thought.
        emitFunctionCheckIfNeeded();
        // We flush everything, as if we were in the backedge of a loop (see treatment of op_jmp in parseBlock).
        flushForTerminal();

        // We must set the callee to the right value
        if (stackEntry->m_inlineCallFrame) {
            if (stackEntry->m_inlineCallFrame->isClosureCall)
                setDirect(stackEntry->remapOperand(VirtualRegister(CallFrameSlot::callee)), callTargetNode, NormalSet);
        } else
            addToGraph(SetCallee, callTargetNode);

        // We must set the arguments to the right values
        if (!stackEntry->m_inlineCallFrame)
            addToGraph(SetArgumentCountIncludingThis, OpInfo(argumentCountIncludingThis));
        int argIndex = 0;
        for (; argIndex < argumentCountIncludingThis; ++argIndex) {
            Node* value = get(virtualRegisterForArgument(argIndex, registerOffset));
            setDirect(stackEntry->remapOperand(virtualRegisterForArgument(argIndex)), value, NormalSet);
        }
        Node* undefined = addToGraph(JSConstant, OpInfo(m_constantUndefined));
        for (; argIndex < stackEntry->m_codeBlock->numParameters(); ++argIndex)
            setDirect(stackEntry->remapOperand(virtualRegisterForArgument(argIndex)), undefined, NormalSet);

        // We must repeat the work of op_enter here as we will jump right after it.
        // We jump right after it and not before it, because of some invariant saying that a CFG root cannot have predecessors in the IR.
        for (int i = 0; i < stackEntry->m_codeBlock->numVars(); ++i)
            setDirect(stackEntry->remapOperand(virtualRegisterForLocal(i)), undefined, NormalSet);

        // We want to emit the SetLocals with an exit origin that points to the place we are jumping to.
        unsigned oldIndex = m_currentIndex;
        auto oldStackTop = m_inlineStackTop;
        m_inlineStackTop = stackEntry;
        m_currentIndex = opcodeLengths[op_enter];
        m_exitOK = true;
        processSetLocalQueue();
        m_currentIndex = oldIndex;
        m_inlineStackTop = oldStackTop;
        m_exitOK = false;

        BasicBlock** entryBlockPtr = tryBinarySearch<BasicBlock*, unsigned>(stackEntry->m_blockLinkingTargets, stackEntry->m_blockLinkingTargets.size(), opcodeLengths[op_enter], getBytecodeBeginForBlock);
        RELEASE_ASSERT(entryBlockPtr);
        addJumpTo(*entryBlockPtr);
        return true;
        // It would be unsound to jump over a non-tail call: the "tail" call is not really a tail call in that case.
    } while (stackEntry->m_inlineCallFrame && stackEntry->m_inlineCallFrame->kind == InlineCallFrame::TailCall && (stackEntry = stackEntry->m_caller));

    // The tail call was not recursive
    return false;
}

unsigned ByteCodeParser::inliningCost(CallVariant callee, int argumentCountIncludingThis, InlineCallFrame::Kind kind)
{
    CallMode callMode = InlineCallFrame::callModeFor(kind);
    CodeSpecializationKind specializationKind = specializationKindFor(callMode);
    VERBOSE_LOG("Considering inlining ", callee, " into ", currentCodeOrigin(), "\n");
    
    if (m_hasDebuggerEnabled) {
        VERBOSE_LOG("    Failing because the debugger is in use.\n");
        return UINT_MAX;
    }

    FunctionExecutable* executable = callee.functionExecutable();
    if (!executable) {
        VERBOSE_LOG("    Failing because there is no function executable.\n");
        return UINT_MAX;
    }
    
    // Do we have a code block, and does the code block's size match the heuristics/requirements for
    // being an inline candidate? We might not have a code block (1) if code was thrown away,
    // (2) if we simply hadn't actually made this call yet or (3) code is a builtin function and
    // specialization kind is construct. In the former 2 cases, we could still theoretically attempt
    // to inline it if we had a static proof of what was being called; this might happen for example
    // if you call a global function, where watchpointing gives us static information. Overall,
    // it's a rare case because we expect that any hot callees would have already been compiled.
    CodeBlock* codeBlock = executable->baselineCodeBlockFor(specializationKind);
    if (!codeBlock) {
        VERBOSE_LOG("    Failing because no code block available.\n");
        return UINT_MAX;
    }

    if (!Options::useArityFixupInlining()) {
        if (codeBlock->numParameters() > argumentCountIncludingThis) {
            VERBOSE_LOG("    Failing because of arity mismatch.\n");
            return UINT_MAX;
        }
    }

    CapabilityLevel capabilityLevel = inlineFunctionForCapabilityLevel(
        codeBlock, specializationKind, callee.isClosureCall());
    VERBOSE_LOG("    Call mode: ", callMode, "\n");
    VERBOSE_LOG("    Is closure call: ", callee.isClosureCall(), "\n");
    VERBOSE_LOG("    Capability level: ", capabilityLevel, "\n");
    VERBOSE_LOG("    Might inline function: ", mightInlineFunctionFor(codeBlock, specializationKind), "\n");
    VERBOSE_LOG("    Might compile function: ", mightCompileFunctionFor(codeBlock, specializationKind), "\n");
    VERBOSE_LOG("    Is supported for inlining: ", isSupportedForInlining(codeBlock), "\n");
    VERBOSE_LOG("    Is inlining candidate: ", codeBlock->ownerExecutable()->isInliningCandidate(), "\n");
    if (!canInline(capabilityLevel)) {
        VERBOSE_LOG("    Failing because the function is not inlineable.\n");
        return UINT_MAX;
    }
    
    // Check if the caller is already too large. We do this check here because that's just
    // where we happen to also have the callee's code block, and we want that for the
    // purpose of unsetting SABI.
    if (!isSmallEnoughToInlineCodeInto(m_codeBlock)) {
        codeBlock->m_shouldAlwaysBeInlined = false;
        VERBOSE_LOG("    Failing because the caller is too large.\n");
        return UINT_MAX;
    }
    
    // FIXME: this should be better at predicting how much bloat we will introduce by inlining
    // this function.
    // https://bugs.webkit.org/show_bug.cgi?id=127627
    
    // FIXME: We currently inline functions that have run in LLInt but not in Baseline. These
    // functions have very low fidelity profiling, and presumably they weren't very hot if they
    // haven't gotten to Baseline yet. Consider not inlining these functions.
    // https://bugs.webkit.org/show_bug.cgi?id=145503
    
    // Have we exceeded inline stack depth, or are we trying to inline a recursive call to
    // too many levels? If either of these are detected, then don't inline. We adjust our
    // heuristics if we are dealing with a function that cannot otherwise be compiled.
    
    unsigned depth = 0;
    unsigned recursion = 0;
    
    for (InlineStackEntry* entry = m_inlineStackTop; entry; entry = entry->m_caller) {
        ++depth;
        if (depth >= Options::maximumInliningDepth()) {
            VERBOSE_LOG("    Failing because depth exceeded.\n");
            return UINT_MAX;
        }
        
        if (entry->executable() == executable) {
            ++recursion;
            if (recursion >= Options::maximumInliningRecursion()) {
                VERBOSE_LOG("    Failing because recursion detected.\n");
                return UINT_MAX;
            }
        }
    }
    
    VERBOSE_LOG("    Inlining should be possible.\n");
    
    // It might be possible to inline.
    return codeBlock->bytecodeCost();
}

template<typename ChecksFunctor>
void ByteCodeParser::inlineCall(Node* callTargetNode, VirtualRegister result, CallVariant callee, int registerOffset, int argumentCountIncludingThis, InlineCallFrame::Kind kind, BasicBlock* continuationBlock, const ChecksFunctor& insertChecks)
{
    const Instruction* savedCurrentInstruction = m_currentInstruction;
    CodeSpecializationKind specializationKind = InlineCallFrame::specializationKindFor(kind);
    
    ASSERT(inliningCost(callee, argumentCountIncludingThis, kind) != UINT_MAX);
    
    CodeBlock* codeBlock = callee.functionExecutable()->baselineCodeBlockFor(specializationKind);
    insertChecks(codeBlock);

    // FIXME: Don't flush constants!

    // arityFixupCount and numberOfStackPaddingSlots are different. While arityFixupCount does not consider about stack alignment,
    // numberOfStackPaddingSlots consider alignment. Consider the following case,
    //
    // before: [ ... ][arg0][header]
    // after:  [ ... ][ext ][arg1][arg0][header]
    //
    // In the above case, arityFixupCount is 1. But numberOfStackPaddingSlots is 2 because the stack needs to be aligned.
    // We insert extra slots to align stack.
    int arityFixupCount = std::max<int>(codeBlock->numParameters() - argumentCountIncludingThis, 0);
    int numberOfStackPaddingSlots = CommonSlowPaths::numberOfStackPaddingSlots(codeBlock, argumentCountIncludingThis);
    ASSERT(!(numberOfStackPaddingSlots % stackAlignmentRegisters()));
    int registerOffsetAfterFixup = registerOffset - numberOfStackPaddingSlots;
    
    int inlineCallFrameStart = m_inlineStackTop->remapOperand(VirtualRegister(registerOffsetAfterFixup)).offset() + CallFrame::headerSizeInRegisters;
    
    ensureLocals(
        VirtualRegister(inlineCallFrameStart).toLocal() + 1 +
        CallFrame::headerSizeInRegisters + codeBlock->numCalleeLocals());
    
    size_t argumentPositionStart = m_graph.m_argumentPositions.size();

    if (result.isValid())
        result = m_inlineStackTop->remapOperand(result);

    VariableAccessData* calleeVariable = nullptr;
    if (callee.isClosureCall()) {
        Node* calleeSet = set(
            VirtualRegister(registerOffsetAfterFixup + CallFrameSlot::callee), callTargetNode, ImmediateNakedSet);
        
        calleeVariable = calleeSet->variableAccessData();
        calleeVariable->mergeShouldNeverUnbox(true);
    }

    InlineStackEntry* callerStackTop = m_inlineStackTop;
    InlineStackEntry inlineStackEntry(this, codeBlock, codeBlock, callee.function(), result,
        (VirtualRegister)inlineCallFrameStart, argumentCountIncludingThis, kind, continuationBlock);

    // This is where the actual inlining really happens.
    unsigned oldIndex = m_currentIndex;
    m_currentIndex = 0;

    switch (kind) {
    case InlineCallFrame::GetterCall:
    case InlineCallFrame::SetterCall: {
        // When inlining getter and setter calls, we setup a stack frame which does not appear in the bytecode.
        // Because Inlining can switch on executable, we could have a graph like this.
        //
        // BB#0
        //     ...
        //     30: GetSetter
        //     31: MovHint(loc10)
        //     32: SetLocal(loc10)
        //     33: MovHint(loc9)
        //     34: SetLocal(loc9)
        //     ...
        //     37: GetExecutable(@30)
        //     ...
        //     41: Switch(@37)
        //
        // BB#2
        //     42: GetLocal(loc12, bc#7 of caller)
        //     ...
        //     --> callee: loc9 and loc10 are arguments of callee.
        //       ...
        //       <HERE, exit to callee, loc9 and loc10 are required in the bytecode>
        //
        // When we prune OSR availability at the beginning of BB#2 (bc#7 in the caller), we prune loc9 and loc10's liveness because the caller does not actually have loc9 and loc10.
        // However, when we begin executing the callee, we need OSR exit to be aware of where it can recover the arguments to the setter, loc9 and loc10. The MovHints in the inlined
        // callee make it so that if we exit at <HERE>, we can recover loc9 and loc10.
        for (int index = 0; index < argumentCountIncludingThis; ++index) {
            VirtualRegister argumentToGet = callerStackTop->remapOperand(virtualRegisterForArgument(index, registerOffset));
            Node* value = getDirect(argumentToGet);
            addToGraph(MovHint, OpInfo(argumentToGet.offset()), value);
            m_setLocalQueue.append(DelayedSetLocal { currentCodeOrigin(), argumentToGet, value, ImmediateNakedSet });
        }
        break;
    }
    default:
        break;
    }

    if (arityFixupCount) {
        // Note: we do arity fixup in two phases:
        // 1. We get all the values we need and MovHint them to the expected locals.
        // 2. We SetLocal them after that. This way, if we exit, the callee's
        //    frame is already set up. If any SetLocal exits, we have a valid exit state.
        //    This is required because if we didn't do this in two phases, we may exit in
        //    the middle of arity fixup from the callee's CodeOrigin. This is unsound because exited
        //    code does not have arity fixup so that remaining necessary fixups are not executed.
        //    For example, consider if we need to pad two args:
        //    [arg3][arg2][arg1][arg0]
        //    [fix ][fix ][arg3][arg2][arg1][arg0]
        //    We memcpy starting from arg0 in the direction of arg3. If we were to exit at a type check
        //    for arg3's SetLocal in the callee's CodeOrigin, we'd exit with a frame like so:
        //    [arg3][arg2][arg1][arg2][arg1][arg0]
        //    Since we do not perform arity fixup in the callee, this is the frame used by the callee.
        //    And the callee would then just end up thinking its argument are:
        //    [fix ][fix ][arg3][arg2][arg1][arg0]
        //    which is incorrect.

        Node* undefined = addToGraph(JSConstant, OpInfo(m_constantUndefined));
        // The stack needs to be aligned due to the JS calling convention. Thus, we have a hole if the count of arguments is not aligned.
        // We call this hole "extra slot". Consider the following case, the number of arguments is 2. If this argument
        // count does not fulfill the stack alignment requirement, we already inserted extra slots.
        //
        // before: [ ... ][ext ][arg1][arg0][header]
        //
        // In the above case, one extra slot is inserted. If the code's parameter count is 3, we will fixup arguments.
        // At that time, we can simply use this extra slots. So the fixuped stack is the following.
        //
        // before: [ ... ][ext ][arg1][arg0][header]
        // after:  [ ... ][arg2][arg1][arg0][header]
        //
        // In such cases, we do not need to move frames.
        if (registerOffsetAfterFixup != registerOffset) {
            for (int index = 0; index < argumentCountIncludingThis; ++index) {
                VirtualRegister argumentToGet = callerStackTop->remapOperand(virtualRegisterForArgument(index, registerOffset));
                Node* value = getDirect(argumentToGet);
                VirtualRegister argumentToSet = m_inlineStackTop->remapOperand(virtualRegisterForArgument(index));
                addToGraph(MovHint, OpInfo(argumentToSet.offset()), value);
                m_setLocalQueue.append(DelayedSetLocal { currentCodeOrigin(), argumentToSet, value, ImmediateNakedSet });
            }
        }
        for (int index = 0; index < arityFixupCount; ++index) {
            VirtualRegister argumentToSet = m_inlineStackTop->remapOperand(virtualRegisterForArgument(argumentCountIncludingThis + index));
            addToGraph(MovHint, OpInfo(argumentToSet.offset()), undefined);
            m_setLocalQueue.append(DelayedSetLocal { currentCodeOrigin(), argumentToSet, undefined, ImmediateNakedSet });
        }

        // At this point, it's OK to OSR exit because we finished setting up
        // our callee's frame. We emit an ExitOK below.
    }

    // At this point, it's again OK to OSR exit.
    m_exitOK = true;
    addToGraph(ExitOK);

    processSetLocalQueue();

    InlineVariableData inlineVariableData;
    inlineVariableData.inlineCallFrame = m_inlineStackTop->m_inlineCallFrame;
    inlineVariableData.argumentPositionStart = argumentPositionStart;
    inlineVariableData.calleeVariable = 0;
    
    RELEASE_ASSERT(
        m_inlineStackTop->m_inlineCallFrame->isClosureCall
        == callee.isClosureCall());
    if (callee.isClosureCall()) {
        RELEASE_ASSERT(calleeVariable);
        inlineVariableData.calleeVariable = calleeVariable;
    }
    
    m_graph.m_inlineVariableData.append(inlineVariableData);

    parseCodeBlock();
    clearCaches(); // Reset our state now that we're back to the outer code.
    
    m_currentIndex = oldIndex;
    m_exitOK = false;

    linkBlocks(inlineStackEntry.m_unlinkedBlocks, inlineStackEntry.m_blockLinkingTargets);
    
    // Most functions have at least one op_ret and thus set up the continuation block.
    // In some rare cases, a function ends in op_unreachable, forcing us to allocate a new continuationBlock here.
    if (inlineStackEntry.m_continuationBlock)
        m_currentBlock = inlineStackEntry.m_continuationBlock;
    else
        m_currentBlock = allocateUntargetableBlock();
    ASSERT(!m_currentBlock->terminal());

    prepareToParseBlock();
    m_currentInstruction = savedCurrentInstruction;
}

ByteCodeParser::CallOptimizationResult ByteCodeParser::handleCallVariant(Node* callTargetNode, VirtualRegister result, CallVariant callee, int registerOffset, VirtualRegister thisArgument, int argumentCountIncludingThis, unsigned nextOffset, InlineCallFrame::Kind kind, SpeculatedType prediction, unsigned& inliningBalance, BasicBlock* continuationBlock, bool needsToCheckCallee)
{
    VERBOSE_LOG("    Considering callee ", callee, "\n");

    bool didInsertChecks = false;
    auto insertChecksWithAccounting = [&] () {
        if (needsToCheckCallee)
            emitFunctionChecks(callee, callTargetNode, thisArgument);
        didInsertChecks = true;
    };

    if (kind == InlineCallFrame::TailCall && ByteCodeParser::handleRecursiveTailCall(callTargetNode, callee, registerOffset, argumentCountIncludingThis, insertChecksWithAccounting)) {
        RELEASE_ASSERT(didInsertChecks);
        return CallOptimizationResult::OptimizedToJump;
    }
    RELEASE_ASSERT(!didInsertChecks);

    if (!inliningBalance)
        return CallOptimizationResult::DidNothing;

    CodeSpecializationKind specializationKind = InlineCallFrame::specializationKindFor(kind);

    auto endSpecialCase = [&] () {
        RELEASE_ASSERT(didInsertChecks);
        addToGraph(Phantom, callTargetNode);
        emitArgumentPhantoms(registerOffset, argumentCountIncludingThis);
        inliningBalance--;
        if (continuationBlock) {
            m_currentIndex = nextOffset;
            m_exitOK = true;
            processSetLocalQueue();
            addJumpTo(continuationBlock);
        }
    };

    if (InternalFunction* function = callee.internalFunction()) {
        if (handleConstantInternalFunction(callTargetNode, result, function, registerOffset, argumentCountIncludingThis, specializationKind, prediction, insertChecksWithAccounting)) {
            endSpecialCase();
            return CallOptimizationResult::Inlined;
        }
        RELEASE_ASSERT(!didInsertChecks);
        return CallOptimizationResult::DidNothing;
    }

    Intrinsic intrinsic = callee.intrinsicFor(specializationKind);
    if (intrinsic != NoIntrinsic) {
        if (handleIntrinsicCall(callTargetNode, result, intrinsic, registerOffset, argumentCountIncludingThis, prediction, insertChecksWithAccounting)) {
            endSpecialCase();
            return CallOptimizationResult::Inlined;
        }
        RELEASE_ASSERT(!didInsertChecks);
        // We might still try to inline the Intrinsic because it might be a builtin JS function.
    }

    if (Options::useDOMJIT()) {
        if (const DOMJIT::Signature* signature = callee.signatureFor(specializationKind)) {
            if (handleDOMJITCall(callTargetNode, result, signature, registerOffset, argumentCountIncludingThis, prediction, insertChecksWithAccounting)) {
                endSpecialCase();
                return CallOptimizationResult::Inlined;
            }
            RELEASE_ASSERT(!didInsertChecks);
        }
    }
    
    unsigned myInliningCost = inliningCost(callee, argumentCountIncludingThis, kind);
    if (myInliningCost > inliningBalance)
        return CallOptimizationResult::DidNothing;

    auto insertCheck = [&] (CodeBlock*) {
        if (needsToCheckCallee)
            emitFunctionChecks(callee, callTargetNode, thisArgument);
    };
    inlineCall(callTargetNode, result, callee, registerOffset, argumentCountIncludingThis, kind, continuationBlock, insertCheck);
    inliningBalance -= myInliningCost;
    return CallOptimizationResult::Inlined;
}

bool ByteCodeParser::handleVarargsInlining(Node* callTargetNode, VirtualRegister result,
    const CallLinkStatus& callLinkStatus, int firstFreeReg, VirtualRegister thisArgument,
    VirtualRegister argumentsArgument, unsigned argumentsOffset,
    NodeType callOp, InlineCallFrame::Kind kind)
{
    VERBOSE_LOG("Handling inlining (Varargs)...\nStack: ", currentCodeOrigin(), "\n");
    if (callLinkStatus.maxNumArguments() > Options::maximumVarargsForInlining()) {
        VERBOSE_LOG("Bailing inlining: too many arguments for varargs inlining.\n");
        return false;
    }
    if (callLinkStatus.couldTakeSlowPath() || callLinkStatus.size() != 1) {
        VERBOSE_LOG("Bailing inlining: polymorphic inlining is not yet supported for varargs.\n");
        return false;
    }

    CallVariant callVariant = callLinkStatus[0];

    unsigned mandatoryMinimum;
    if (FunctionExecutable* functionExecutable = callVariant.functionExecutable())
        mandatoryMinimum = functionExecutable->parameterCount();
    else
        mandatoryMinimum = 0;
    
    // includes "this"
    unsigned maxNumArguments = std::max(callLinkStatus.maxNumArguments(), mandatoryMinimum + 1);

    CodeSpecializationKind specializationKind = InlineCallFrame::specializationKindFor(kind);
    if (inliningCost(callVariant, maxNumArguments, kind) > getInliningBalance(callLinkStatus, specializationKind)) {
        VERBOSE_LOG("Bailing inlining: inlining cost too high.\n");
        return false;
    }
    
    int registerOffset = firstFreeReg + 1;
    registerOffset -= maxNumArguments; // includes "this"
    registerOffset -= CallFrame::headerSizeInRegisters;
    registerOffset = -WTF::roundUpToMultipleOf(stackAlignmentRegisters(), -registerOffset);

    auto insertChecks = [&] (CodeBlock* codeBlock) {
        emitFunctionChecks(callVariant, callTargetNode, thisArgument);
        
        int remappedRegisterOffset =
        m_inlineStackTop->remapOperand(VirtualRegister(registerOffset)).offset();
        
        ensureLocals(VirtualRegister(remappedRegisterOffset).toLocal());
        
        int argumentStart = registerOffset + CallFrame::headerSizeInRegisters;
        int remappedArgumentStart = m_inlineStackTop->remapOperand(VirtualRegister(argumentStart)).offset();
        
        LoadVarargsData* data = m_graph.m_loadVarargsData.add();
        data->start = VirtualRegister(remappedArgumentStart + 1);
        data->count = VirtualRegister(remappedRegisterOffset + CallFrameSlot::argumentCount);
        data->offset = argumentsOffset;
        data->limit = maxNumArguments;
        data->mandatoryMinimum = mandatoryMinimum;
        
        if (callOp == TailCallForwardVarargs)
            addToGraph(ForwardVarargs, OpInfo(data));
        else
            addToGraph(LoadVarargs, OpInfo(data), get(argumentsArgument));
        
        // LoadVarargs may OSR exit. Hence, we need to keep alive callTargetNode, thisArgument
        // and argumentsArgument for the baseline JIT. However, we only need a Phantom for
        // callTargetNode because the other 2 are still in use and alive at this point.
        addToGraph(Phantom, callTargetNode);
        
        // In DFG IR before SSA, we cannot insert control flow between after the
        // LoadVarargs and the last SetArgumentDefinitely. This isn't a problem once we get to DFG
        // SSA. Fortunately, we also have other reasons for not inserting control flow
        // before SSA.
        
        VariableAccessData* countVariable = newVariableAccessData(VirtualRegister(remappedRegisterOffset + CallFrameSlot::argumentCount));
        // This is pretty lame, but it will force the count to be flushed as an int. This doesn't
        // matter very much, since our use of a SetArgumentDefinitely and Flushes for this local slot is
        // mostly just a formality.
        countVariable->predict(SpecInt32Only);
        countVariable->mergeIsProfitableToUnbox(true);
        Node* setArgumentCount = addToGraph(SetArgumentDefinitely, OpInfo(countVariable));
        m_currentBlock->variablesAtTail.setOperand(countVariable->local(), setArgumentCount);
        
        set(VirtualRegister(argumentStart), get(thisArgument), ImmediateNakedSet);
        unsigned numSetArguments = 0;
        for (unsigned argument = 1; argument < maxNumArguments; ++argument) {
            VariableAccessData* variable = newVariableAccessData(VirtualRegister(remappedArgumentStart + argument));
            variable->mergeShouldNeverUnbox(true); // We currently have nowhere to put the type check on the LoadVarargs. LoadVarargs is effectful, so after it finishes, we cannot exit.
            
            // For a while it had been my intention to do things like this inside the
            // prediction injection phase. But in this case it's really best to do it here,
            // because it's here that we have access to the variable access datas for the
            // inlining we're about to do.
            //
            // Something else that's interesting here is that we'd really love to get
            // predictions from the arguments loaded at the callsite, rather than the
            // arguments received inside the callee. But that probably won't matter for most
            // calls.
            if (codeBlock && argument < static_cast<unsigned>(codeBlock->numParameters())) {
                ConcurrentJSLocker locker(codeBlock->m_lock);
                ValueProfile& profile = codeBlock->valueProfileForArgument(argument);
                variable->predict(profile.computeUpdatedPrediction(locker));
            }
            
            Node* setArgument = addToGraph(numSetArguments >= mandatoryMinimum ? SetArgumentMaybe : SetArgumentDefinitely, OpInfo(variable));
            m_currentBlock->variablesAtTail.setOperand(variable->local(), setArgument);
            ++numSetArguments;
        }
    };

    // Intrinsics and internal functions can only be inlined if we're not doing varargs. This is because
    // we currently don't have any way of getting profiling information for arguments to non-JS varargs
    // calls. The prediction propagator won't be of any help because LoadVarargs obscures the data flow,
    // and there are no callsite value profiles and native function won't have callee value profiles for
    // those arguments. Even worse, if the intrinsic decides to exit, it won't really have anywhere to
    // exit to: LoadVarargs is effectful and it's part of the op_call_varargs, so we can't exit without
    // calling LoadVarargs twice.
    inlineCall(callTargetNode, result, callVariant, registerOffset, maxNumArguments, kind, nullptr, insertChecks);


    VERBOSE_LOG("Successful inlining (varargs, monomorphic).\nStack: ", currentCodeOrigin(), "\n");
    return true;
}

unsigned ByteCodeParser::getInliningBalance(const CallLinkStatus& callLinkStatus, CodeSpecializationKind specializationKind)
{
    unsigned inliningBalance = Options::maximumFunctionForCallInlineCandidateBytecodeCost();
    if (specializationKind == CodeForConstruct)
        inliningBalance = std::min(inliningBalance, Options::maximumFunctionForConstructInlineCandidateBytecoodeCost());
    if (callLinkStatus.isClosureCall())
        inliningBalance = std::min(inliningBalance, Options::maximumFunctionForClosureCallInlineCandidateBytecodeCost());
    return inliningBalance;
}

ByteCodeParser::CallOptimizationResult ByteCodeParser::handleInlining(
    Node* callTargetNode, VirtualRegister result, const CallLinkStatus& callLinkStatus,
    int registerOffset, VirtualRegister thisArgument,
    int argumentCountIncludingThis,
    unsigned nextOffset, NodeType callOp, InlineCallFrame::Kind kind, SpeculatedType prediction)
{
    VERBOSE_LOG("Handling inlining...\nStack: ", currentCodeOrigin(), "\n");
    
    CodeSpecializationKind specializationKind = InlineCallFrame::specializationKindFor(kind);
    unsigned inliningBalance = getInliningBalance(callLinkStatus, specializationKind);

    // First check if we can avoid creating control flow. Our inliner does some CFG
    // simplification on the fly and this helps reduce compile times, but we can only leverage
    // this in cases where we don't need control flow diamonds to check the callee.
    if (!callLinkStatus.couldTakeSlowPath() && callLinkStatus.size() == 1) {
        return handleCallVariant(
            callTargetNode, result, callLinkStatus[0], registerOffset, thisArgument,
            argumentCountIncludingThis, nextOffset, kind, prediction, inliningBalance, nullptr, true);
    }

    // We need to create some kind of switch over callee. For now we only do this if we believe that
    // we're in the top tier. We have two reasons for this: first, it provides us an opportunity to
    // do more detailed polyvariant/polymorphic profiling; and second, it reduces compile times in
    // the DFG. And by polyvariant profiling we mean polyvariant profiling of *this* call. Note that
    // we could improve that aspect of this by doing polymorphic inlining but having the profiling
    // also.
    if (!m_graph.m_plan.isFTL() || !Options::usePolymorphicCallInlining()) {
        VERBOSE_LOG("Bailing inlining (hard).\nStack: ", currentCodeOrigin(), "\n");
        return CallOptimizationResult::DidNothing;
    }
    
    // If the claim is that this did not originate from a stub, then we don't want to emit a switch
    // statement. Whenever the non-stub profiling says that it could take slow path, it really means that
    // it has no idea.
    if (!Options::usePolymorphicCallInliningForNonStubStatus()
        && !callLinkStatus.isBasedOnStub()) {
        VERBOSE_LOG("Bailing inlining (non-stub polymorphism).\nStack: ", currentCodeOrigin(), "\n");
        return CallOptimizationResult::DidNothing;
    }

    bool allAreClosureCalls = true;
    bool allAreDirectCalls = true;
    for (unsigned i = callLinkStatus.size(); i--;) {
        if (callLinkStatus[i].isClosureCall())
            allAreDirectCalls = false;
        else
            allAreClosureCalls = false;
    }

    Node* thingToSwitchOn;
    if (allAreDirectCalls)
        thingToSwitchOn = callTargetNode;
    else if (allAreClosureCalls)
        thingToSwitchOn = addToGraph(GetExecutable, callTargetNode);
    else {
        // FIXME: We should be able to handle this case, but it's tricky and we don't know of cases
        // where it would be beneficial. It might be best to handle these cases as if all calls were
        // closure calls.
        // https://bugs.webkit.org/show_bug.cgi?id=136020
        VERBOSE_LOG("Bailing inlining (mix).\nStack: ", currentCodeOrigin(), "\n");
        return CallOptimizationResult::DidNothing;
    }

    VERBOSE_LOG("Doing hard inlining...\nStack: ", currentCodeOrigin(), "\n");

    // This makes me wish that we were in SSA all the time. We need to pick a variable into which to
    // store the callee so that it will be accessible to all of the blocks we're about to create. We
    // get away with doing an immediate-set here because we wouldn't have performed any side effects
    // yet.
    VERBOSE_LOG("Register offset: ", registerOffset);
    VirtualRegister calleeReg(registerOffset + CallFrameSlot::callee);
    calleeReg = m_inlineStackTop->remapOperand(calleeReg);
    VERBOSE_LOG("Callee is going to be ", calleeReg, "\n");
    setDirect(calleeReg, callTargetNode, ImmediateSetWithFlush);

    // It's OK to exit right now, even though we set some locals. That's because those locals are not
    // user-visible.
    m_exitOK = true;
    addToGraph(ExitOK);
    
    SwitchData& data = *m_graph.m_switchData.add();
    data.kind = SwitchCell;
    addToGraph(Switch, OpInfo(&data), thingToSwitchOn);
    m_currentBlock->didLink();
    
    BasicBlock* continuationBlock = allocateUntargetableBlock();
    VERBOSE_LOG("Adding untargetable block ", RawPointer(continuationBlock), " (continuation)\n");
    
    // We may force this true if we give up on inlining any of the edges.
    bool couldTakeSlowPath = callLinkStatus.couldTakeSlowPath();
    
    VERBOSE_LOG("About to loop over functions at ", currentCodeOrigin(), ".\n");

    unsigned oldOffset = m_currentIndex;
    for (unsigned i = 0; i < callLinkStatus.size(); ++i) {
        m_currentIndex = oldOffset;
        BasicBlock* calleeEntryBlock = allocateUntargetableBlock();
        m_currentBlock = calleeEntryBlock;
        prepareToParseBlock();

        // At the top of each switch case, we can exit.
        m_exitOK = true;
        
        Node* myCallTargetNode = getDirect(calleeReg);
        
        auto inliningResult = handleCallVariant(
            myCallTargetNode, result, callLinkStatus[i], registerOffset,
            thisArgument, argumentCountIncludingThis, nextOffset, kind, prediction,
            inliningBalance, continuationBlock, false);
        
        if (inliningResult == CallOptimizationResult::DidNothing) {
            // That failed so we let the block die. Nothing interesting should have been added to
            // the block. We also give up on inlining any of the (less frequent) callees.
            ASSERT(m_graph.m_blocks.last() == m_currentBlock);
            m_graph.killBlockAndItsContents(m_currentBlock);
            m_graph.m_blocks.removeLast();
            VERBOSE_LOG("Inlining of a poly call failed, we will have to go through a slow path\n");

            // The fact that inlining failed means we need a slow path.
            couldTakeSlowPath = true;
            break;
        }
        
        JSCell* thingToCaseOn;
        if (allAreDirectCalls)
            thingToCaseOn = callLinkStatus[i].nonExecutableCallee();
        else {
            ASSERT(allAreClosureCalls);
            thingToCaseOn = callLinkStatus[i].executable();
        }
        data.cases.append(SwitchCase(m_graph.freeze(thingToCaseOn), calleeEntryBlock));
        VERBOSE_LOG("Finished optimizing ", callLinkStatus[i], " at ", currentCodeOrigin(), ".\n");
    }

    // Slow path block
    m_currentBlock = allocateUntargetableBlock();
    m_currentIndex = oldOffset;
    m_exitOK = true;
    data.fallThrough = BranchTarget(m_currentBlock);
    prepareToParseBlock();
    Node* myCallTargetNode = getDirect(calleeReg);
    if (couldTakeSlowPath) {
        addCall(
            result, callOp, nullptr, myCallTargetNode, argumentCountIncludingThis,
            registerOffset, prediction);
        VERBOSE_LOG("We added a call in the slow path\n");
    } else {
        addToGraph(CheckBadCell);
        addToGraph(Phantom, myCallTargetNode);
        emitArgumentPhantoms(registerOffset, argumentCountIncludingThis);
        
        set(result, addToGraph(BottomValue));
        VERBOSE_LOG("couldTakeSlowPath was false\n");
    }

    m_currentIndex = nextOffset;
    m_exitOK = true; // Origin changed, so it's fine to exit again.
    processSetLocalQueue();

    if (Node* terminal = m_currentBlock->terminal())
        ASSERT_UNUSED(terminal, terminal->op() == TailCall || terminal->op() == TailCallVarargs || terminal->op() == TailCallForwardVarargs);
    else {
        addJumpTo(continuationBlock);
    }

    prepareToParseBlock();
    
    m_currentIndex = oldOffset;
    m_currentBlock = continuationBlock;
    m_exitOK = true;
    
    VERBOSE_LOG("Done inlining (hard).\nStack: ", currentCodeOrigin(), "\n");
    return CallOptimizationResult::Inlined;
}

template<typename ChecksFunctor>
bool ByteCodeParser::handleMinMax(VirtualRegister result, NodeType op, int registerOffset, int argumentCountIncludingThis, const ChecksFunctor& insertChecks)
{
    ASSERT(op == ArithMin || op == ArithMax);

    if (argumentCountIncludingThis == 1) {
        insertChecks();
        double limit = op == ArithMax ? -std::numeric_limits<double>::infinity() : +std::numeric_limits<double>::infinity();
        set(result, addToGraph(JSConstant, OpInfo(m_graph.freeze(jsDoubleNumber(limit)))));
        return true;
    }
     
    if (argumentCountIncludingThis == 2) {
        insertChecks();
        Node* resultNode = get(VirtualRegister(virtualRegisterForArgument(1, registerOffset)));
        addToGraph(Phantom, Edge(resultNode, NumberUse));
        set(result, resultNode);
        return true;
    }
    
    if (argumentCountIncludingThis == 3) {
        insertChecks();
        set(result, addToGraph(op, get(virtualRegisterForArgument(1, registerOffset)), get(virtualRegisterForArgument(2, registerOffset))));
        return true;
    }
    
    // Don't handle >=3 arguments for now.
    return false;
}

template<typename ChecksFunctor>
bool ByteCodeParser::handleIntrinsicCall(Node* callee, VirtualRegister result, Intrinsic intrinsic, int registerOffset, int argumentCountIncludingThis, SpeculatedType prediction, const ChecksFunctor& insertChecks)
{
    VERBOSE_LOG("       The intrinsic is ", intrinsic, "\n");

    if (!isOpcodeShape<OpCallShape>(m_currentInstruction))
        return false;

    // It so happens that the code below doesn't handle the invalid result case. We could fix that, but
    // it would only benefit intrinsics called as setters, like if you do:
    //
    //     o.__defineSetter__("foo", Math.pow)
    //
    // Which is extremely amusing, but probably not worth optimizing.
    if (!result.isValid())
        return false;

    bool didSetResult = false;
    auto setResult = [&] (Node* node) {
        RELEASE_ASSERT(!didSetResult);
        set(result, node);
        didSetResult = true;
    };

    auto inlineIntrinsic = [&] {
        switch (intrinsic) {

        // Intrinsic Functions:

        case AbsIntrinsic: {
            if (argumentCountIncludingThis == 1) { // Math.abs()
                insertChecks();
                setResult(addToGraph(JSConstant, OpInfo(m_constantNaN)));
                return true;
            }

            if (!MacroAssembler::supportsFloatingPointAbs())
                return false;

            insertChecks();
            Node* node = addToGraph(ArithAbs, get(virtualRegisterForArgument(1, registerOffset)));
            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, Overflow))
                node->mergeFlags(NodeMayOverflowInt32InDFG);
            setResult(node);
            return true;
        }

        case MinIntrinsic:
        case MaxIntrinsic:
            if (handleMinMax(result, intrinsic == MinIntrinsic ? ArithMin : ArithMax, registerOffset, argumentCountIncludingThis, insertChecks)) {
                didSetResult = true;
                return true;
            }
            return false;

#define DFG_ARITH_UNARY(capitalizedName, lowerName) \
        case capitalizedName##Intrinsic:
        FOR_EACH_DFG_ARITH_UNARY_OP(DFG_ARITH_UNARY)
#undef DFG_ARITH_UNARY
        {
            if (argumentCountIncludingThis == 1) {
                insertChecks();
                setResult(addToGraph(JSConstant, OpInfo(m_constantNaN)));
                return true;
            }
            Arith::UnaryType type = Arith::UnaryType::Sin;
            switch (intrinsic) {
#define DFG_ARITH_UNARY(capitalizedName, lowerName) \
            case capitalizedName##Intrinsic: \
                type = Arith::UnaryType::capitalizedName; \
                break;
        FOR_EACH_DFG_ARITH_UNARY_OP(DFG_ARITH_UNARY)
#undef DFG_ARITH_UNARY
            default:
                RELEASE_ASSERT_NOT_REACHED();
            }
            insertChecks();
            setResult(addToGraph(ArithUnary, OpInfo(static_cast<std::underlying_type<Arith::UnaryType>::type>(type)), get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case FRoundIntrinsic:
        case SqrtIntrinsic: {
            if (argumentCountIncludingThis == 1) {
                insertChecks();
                setResult(addToGraph(JSConstant, OpInfo(m_constantNaN)));
                return true;
            }

            NodeType nodeType = Unreachable;
            switch (intrinsic) {
            case FRoundIntrinsic:
                nodeType = ArithFRound;
                break;
            case SqrtIntrinsic:
                nodeType = ArithSqrt;
                break;
            default:
                RELEASE_ASSERT_NOT_REACHED();
            }
            insertChecks();
            setResult(addToGraph(nodeType, get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case PowIntrinsic: {
            if (argumentCountIncludingThis < 3) {
                // Math.pow() and Math.pow(x) return NaN.
                insertChecks();
                setResult(addToGraph(JSConstant, OpInfo(m_constantNaN)));
                return true;
            }
            insertChecks();
            VirtualRegister xOperand = virtualRegisterForArgument(1, registerOffset);
            VirtualRegister yOperand = virtualRegisterForArgument(2, registerOffset);
            setResult(addToGraph(ArithPow, get(xOperand), get(yOperand)));
            return true;
        }
            
        case ArrayPushIntrinsic: {
#if USE(JSVALUE32_64)
            if (isX86()) {
                if (argumentCountIncludingThis > 2)
                    return false;
            }
#endif

            if (static_cast<unsigned>(argumentCountIncludingThis) >= MIN_SPARSE_ARRAY_INDEX)
                return false;
            
            ArrayMode arrayMode = getArrayMode(Array::Write);
            if (!arrayMode.isJSArray())
                return false;
            switch (arrayMode.type()) {
            case Array::Int32:
            case Array::Double:
            case Array::Contiguous:
            case Array::ArrayStorage: {
                insertChecks();

                addVarArgChild(nullptr); // For storage.
                for (int i = 0; i < argumentCountIncludingThis; ++i)
                    addVarArgChild(get(virtualRegisterForArgument(i, registerOffset)));
                Node* arrayPush = addToGraph(Node::VarArg, ArrayPush, OpInfo(arrayMode.asWord()), OpInfo(prediction));
                setResult(arrayPush);
                return true;
            }
                
            default:
                return false;
            }
        }

        case ArraySliceIntrinsic: {
#if USE(JSVALUE32_64)
            if (isX86()) {
                // There aren't enough registers for this to be done easily.
                return false;
            }
#endif
            if (argumentCountIncludingThis < 1)
                return false;

            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadConstantCache)
                || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadCache))
                return false;

            ArrayMode arrayMode = getArrayMode(Array::Read);
            if (!arrayMode.isJSArray())
                return false;

            if (!arrayMode.isJSArrayWithOriginalStructure())
                return false;

            switch (arrayMode.type()) {
            case Array::Double:
            case Array::Int32:
            case Array::Contiguous: {
                JSGlobalObject* globalObject = m_graph.globalObjectFor(currentNodeOrigin().semantic);

                Structure* arrayPrototypeStructure = globalObject->arrayPrototype()->structure(*m_vm);
                Structure* objectPrototypeStructure = globalObject->objectPrototype()->structure(*m_vm);

                // FIXME: We could easily relax the Array/Object.prototype transition as long as we OSR exitted if we saw a hole.
                // https://bugs.webkit.org/show_bug.cgi?id=173171
                if (globalObject->arraySpeciesWatchpointSet().state() == IsWatched
                    && globalObject->havingABadTimeWatchpoint()->isStillValid()
                    && arrayPrototypeStructure->transitionWatchpointSetIsStillValid()
                    && objectPrototypeStructure->transitionWatchpointSetIsStillValid()
                    && globalObject->arrayPrototypeChainIsSane()) {

                    m_graph.watchpoints().addLazily(globalObject->arraySpeciesWatchpointSet());
                    m_graph.watchpoints().addLazily(globalObject->havingABadTimeWatchpoint());
                    m_graph.registerAndWatchStructureTransition(arrayPrototypeStructure);
                    m_graph.registerAndWatchStructureTransition(objectPrototypeStructure);

                    insertChecks();

                    Node* array = get(virtualRegisterForArgument(0, registerOffset));
                    // We do a few things here to prove that we aren't skipping doing side-effects in an observable way:
                    // 1. We ensure that the "constructor" property hasn't been changed (because the observable
                    // effects of slice require that we perform a Get(array, "constructor") and we can skip
                    // that if we're an original array structure. (We can relax this in the future by using
                    // TryGetById and CheckCell).
                    //
                    // 2. We check that the array we're calling slice on has the same global object as the lexical
                    // global object that this code is running in. This requirement is necessary because we setup the
                    // watchpoints above on the lexical global object. This means that code that calls slice on
                    // arrays produced by other global objects won't get this optimization. We could relax this
                    // requirement in the future by checking that the watchpoint hasn't fired at runtime in the code
                    // we generate instead of registering it as a watchpoint that would invalidate the compilation.
                    //
                    // 3. By proving we're an original array structure, we guarantee that the incoming array
                    // isn't a subclass of Array.

                    StructureSet structureSet;
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(ArrayWithInt32));
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(ArrayWithContiguous));
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(ArrayWithDouble));
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(CopyOnWriteArrayWithInt32));
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(CopyOnWriteArrayWithContiguous));
                    structureSet.add(globalObject->originalArrayStructureForIndexingType(CopyOnWriteArrayWithDouble));
                    addToGraph(CheckStructure, OpInfo(m_graph.addStructureSet(structureSet)), array);

                    addVarArgChild(array);
                    if (argumentCountIncludingThis >= 2)
                        addVarArgChild(get(virtualRegisterForArgument(1, registerOffset))); // Start index.
                    if (argumentCountIncludingThis >= 3)
                        addVarArgChild(get(virtualRegisterForArgument(2, registerOffset))); // End index.
                    addVarArgChild(addToGraph(GetButterfly, array));

                    Node* arraySlice = addToGraph(Node::VarArg, ArraySlice, OpInfo(), OpInfo());
                    setResult(arraySlice);
                    return true;
                }

                return false;
            }
            default:
                return false;
            }

            RELEASE_ASSERT_NOT_REACHED();
            return false;
        }

        case ArrayIndexOfIntrinsic: {
            if (argumentCountIncludingThis < 2)
                return false;

            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadIndexingType)
                || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadConstantCache)
                || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadCache)
                || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadType))
                return false;

            ArrayMode arrayMode = getArrayMode(Array::Read);
            if (!arrayMode.isJSArray())
                return false;

            if (!arrayMode.isJSArrayWithOriginalStructure())
                return false;

            // We do not want to convert arrays into one type just to perform indexOf.
            if (arrayMode.doesConversion())
                return false;

            switch (arrayMode.type()) {
            case Array::Double:
            case Array::Int32:
            case Array::Contiguous: {
                JSGlobalObject* globalObject = m_graph.globalObjectFor(currentNodeOrigin().semantic);

                Structure* arrayPrototypeStructure = globalObject->arrayPrototype()->structure(*m_vm);
                Structure* objectPrototypeStructure = globalObject->objectPrototype()->structure(*m_vm);

                // FIXME: We could easily relax the Array/Object.prototype transition as long as we OSR exitted if we saw a hole.
                // https://bugs.webkit.org/show_bug.cgi?id=173171
                if (arrayPrototypeStructure->transitionWatchpointSetIsStillValid()
                    && objectPrototypeStructure->transitionWatchpointSetIsStillValid()
                    && globalObject->arrayPrototypeChainIsSane()) {

                    m_graph.registerAndWatchStructureTransition(arrayPrototypeStructure);
                    m_graph.registerAndWatchStructureTransition(objectPrototypeStructure);

                    insertChecks();

                    Node* array = get(virtualRegisterForArgument(0, registerOffset));
                    addVarArgChild(array);
                    addVarArgChild(get(virtualRegisterForArgument(1, registerOffset))); // Search element.
                    if (argumentCountIncludingThis >= 3)
                        addVarArgChild(get(virtualRegisterForArgument(2, registerOffset))); // Start index.
                    addVarArgChild(nullptr);

                    Node* node = addToGraph(Node::VarArg, ArrayIndexOf, OpInfo(arrayMode.asWord()), OpInfo());
                    setResult(node);
                    return true;
                }

                return false;
            }
            default:
                return false;
            }

            RELEASE_ASSERT_NOT_REACHED();
            return false;

        }
            
        case ArrayPopIntrinsic: {
            if (argumentCountIncludingThis != 1)
                return false;
            
            ArrayMode arrayMode = getArrayMode(Array::Write);
            if (!arrayMode.isJSArray())
                return false;
            switch (arrayMode.type()) {
            case Array::Int32:
            case Array::Double:
            case Array::Contiguous:
            case Array::ArrayStorage: {
                insertChecks();
                Node* arrayPop = addToGraph(ArrayPop, OpInfo(arrayMode.asWord()), OpInfo(prediction), get(virtualRegisterForArgument(0, registerOffset)));
                setResult(arrayPop);
                return true;
            }
                
            default:
                return false;
            }
        }
            
        case AtomicsAddIntrinsic:
        case AtomicsAndIntrinsic:
        case AtomicsCompareExchangeIntrinsic:
        case AtomicsExchangeIntrinsic:
        case AtomicsIsLockFreeIntrinsic:
        case AtomicsLoadIntrinsic:
        case AtomicsOrIntrinsic:
        case AtomicsStoreIntrinsic:
        case AtomicsSubIntrinsic:
        case AtomicsXorIntrinsic: {
            if (!is64Bit())
                return false;
            
            NodeType op = LastNodeType;
            Array::Action action = Array::Write;
            unsigned numArgs = 0; // Number of actual args; we add one for the backing store pointer.
            switch (intrinsic) {
            case AtomicsAddIntrinsic:
                op = AtomicsAdd;
                numArgs = 3;
                break;
            case AtomicsAndIntrinsic:
                op = AtomicsAnd;
                numArgs = 3;
                break;
            case AtomicsCompareExchangeIntrinsic:
                op = AtomicsCompareExchange;
                numArgs = 4;
                break;
            case AtomicsExchangeIntrinsic:
                op = AtomicsExchange;
                numArgs = 3;
                break;
            case AtomicsIsLockFreeIntrinsic:
                // This gets no backing store, but we need no special logic for this since this also does
                // not need varargs.
                op = AtomicsIsLockFree;
                numArgs = 1;
                break;
            case AtomicsLoadIntrinsic:
                op = AtomicsLoad;
                numArgs = 2;
                action = Array::Read;
                break;
            case AtomicsOrIntrinsic:
                op = AtomicsOr;
                numArgs = 3;
                break;
            case AtomicsStoreIntrinsic:
                op = AtomicsStore;
                numArgs = 3;
                break;
            case AtomicsSubIntrinsic:
                op = AtomicsSub;
                numArgs = 3;
                break;
            case AtomicsXorIntrinsic:
                op = AtomicsXor;
                numArgs = 3;
                break;
            default:
                RELEASE_ASSERT_NOT_REACHED();
                break;
            }
            
            if (static_cast<unsigned>(argumentCountIncludingThis) < 1 + numArgs)
                return false;
            
            insertChecks();
            
            Vector<Node*, 3> args;
            for (unsigned i = 0; i < numArgs; ++i)
                args.append(get(virtualRegisterForArgument(1 + i, registerOffset)));
            
            Node* resultNode;
            if (numArgs + 1 <= 3) {
                while (args.size() < 3)
                    args.append(nullptr);
                resultNode = addToGraph(op, OpInfo(ArrayMode(Array::SelectUsingPredictions, action).asWord()), OpInfo(prediction), args[0], args[1], args[2]);
            } else {
                for (Node* node : args)
                    addVarArgChild(node);
                addVarArgChild(nullptr);
                resultNode = addToGraph(Node::VarArg, op, OpInfo(ArrayMode(Array::SelectUsingPredictions, action).asWord()), OpInfo(prediction));
            }
            
            setResult(resultNode);
            return true;
        }

        case ParseIntIntrinsic: {
            if (argumentCountIncludingThis < 2)
                return false;

            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadCell) || m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadType))
                return false;

            insertChecks();
            VirtualRegister valueOperand = virtualRegisterForArgument(1, registerOffset);
            Node* parseInt;
            if (argumentCountIncludingThis == 2)
                parseInt = addToGraph(ParseInt, OpInfo(), OpInfo(prediction), get(valueOperand));
            else {
                ASSERT(argumentCountIncludingThis > 2);
                VirtualRegister radixOperand = virtualRegisterForArgument(2, registerOffset);
                parseInt = addToGraph(ParseInt, OpInfo(), OpInfo(prediction), get(valueOperand), get(radixOperand));
            }
            setResult(parseInt);
            return true;
        }

        case CharCodeAtIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            VirtualRegister thisOperand = virtualRegisterForArgument(0, registerOffset);
            VirtualRegister indexOperand = virtualRegisterForArgument(1, registerOffset);
            Node* charCode = addToGraph(StringCharCodeAt, OpInfo(ArrayMode(Array::String, Array::Read).asWord()), get(thisOperand), get(indexOperand));

            setResult(charCode);
            return true;
        }

        case CharAtIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            VirtualRegister thisOperand = virtualRegisterForArgument(0, registerOffset);
            VirtualRegister indexOperand = virtualRegisterForArgument(1, registerOffset);
            Node* charCode = addToGraph(StringCharAt, OpInfo(ArrayMode(Array::String, Array::Read).asWord()), get(thisOperand), get(indexOperand));

            setResult(charCode);
            return true;
        }
        case Clz32Intrinsic: {
            insertChecks();
            if (argumentCountIncludingThis == 1)
                setResult(addToGraph(JSConstant, OpInfo(m_graph.freeze(jsNumber(32)))));
            else {
                Node* operand = get(virtualRegisterForArgument(1, registerOffset));
                setResult(addToGraph(ArithClz32, operand));
            }
            return true;
        }
        case FromCharCodeIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            VirtualRegister indexOperand = virtualRegisterForArgument(1, registerOffset);
            Node* charCode = addToGraph(StringFromCharCode, get(indexOperand));

            setResult(charCode);

            return true;
        }

        case RegExpExecIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;
            
            insertChecks();
            Node* regExpExec = addToGraph(RegExpExec, OpInfo(0), OpInfo(prediction), addToGraph(GetGlobalObject, callee), get(virtualRegisterForArgument(0, registerOffset)), get(virtualRegisterForArgument(1, registerOffset)));
            setResult(regExpExec);
            
            return true;
        }
            
        case RegExpTestIntrinsic:
        case RegExpTestFastIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            if (intrinsic == RegExpTestIntrinsic) {
                // Don't inline intrinsic if we exited due to one of the primordial RegExp checks failing.
                if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadCell))
                    return false;

                JSGlobalObject* globalObject = m_inlineStackTop->m_codeBlock->globalObject();
                Structure* regExpStructure = globalObject->regExpStructure();
                m_graph.registerStructure(regExpStructure);
                ASSERT(regExpStructure->storedPrototype().isObject());
                ASSERT(regExpStructure->storedPrototype().asCell()->classInfo(*m_vm) == RegExpPrototype::info());

                FrozenValue* regExpPrototypeObjectValue = m_graph.freeze(regExpStructure->storedPrototype());
                Structure* regExpPrototypeStructure = regExpPrototypeObjectValue->structure();

                auto isRegExpPropertySame = [&] (JSValue primordialProperty, UniquedStringImpl* propertyUID) {
                    JSValue currentProperty;
                    if (!m_graph.getRegExpPrototypeProperty(regExpStructure->storedPrototypeObject(), regExpPrototypeStructure, propertyUID, currentProperty))
                        return false;
                    
                    return currentProperty == primordialProperty;
                };

                // Check that RegExp.exec is still the primordial RegExp.prototype.exec
                if (!isRegExpPropertySame(globalObject->regExpProtoExecFunction(), m_vm->propertyNames->exec.impl()))
                    return false;

                // Check that regExpObject is actually a RegExp object.
                Node* regExpObject = get(virtualRegisterForArgument(0, registerOffset));
                addToGraph(Check, Edge(regExpObject, RegExpObjectUse));

                // Check that regExpObject's exec is actually the primodial RegExp.prototype.exec.
                UniquedStringImpl* execPropertyID = m_vm->propertyNames->exec.impl();
                unsigned execIndex = m_graph.identifiers().ensure(execPropertyID);
                Node* actualProperty = addToGraph(TryGetById, OpInfo(execIndex), OpInfo(SpecFunction), Edge(regExpObject, CellUse));
                FrozenValue* regExpPrototypeExec = m_graph.freeze(globalObject->regExpProtoExecFunction());
                addToGraph(CheckCell, OpInfo(regExpPrototypeExec), Edge(actualProperty, CellUse));
            }

            insertChecks();
            Node* regExpObject = get(virtualRegisterForArgument(0, registerOffset));
            Node* regExpExec = addToGraph(RegExpTest, OpInfo(0), OpInfo(prediction), addToGraph(GetGlobalObject, callee), regExpObject, get(virtualRegisterForArgument(1, registerOffset)));
            setResult(regExpExec);
            
            return true;
        }

        case RegExpMatchFastIntrinsic: {
            RELEASE_ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            Node* regExpMatch = addToGraph(RegExpMatchFast, OpInfo(0), OpInfo(prediction), addToGraph(GetGlobalObject, callee), get(virtualRegisterForArgument(0, registerOffset)), get(virtualRegisterForArgument(1, registerOffset)));
            setResult(regExpMatch);
            return true;
        }

        case ObjectCreateIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            setResult(addToGraph(ObjectCreate, get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case ObjectGetPrototypeOfIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            setResult(addToGraph(GetPrototypeOf, OpInfo(0), OpInfo(prediction), get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case ObjectIsIntrinsic: {
            if (argumentCountIncludingThis < 3)
                return false;

            insertChecks();
            setResult(addToGraph(SameValue, get(virtualRegisterForArgument(1, registerOffset)), get(virtualRegisterForArgument(2, registerOffset))));
            return true;
        }

        case ObjectKeysIntrinsic: {
            if (argumentCountIncludingThis < 2)
                return false;

            insertChecks();
            setResult(addToGraph(ObjectKeys, get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case ReflectGetPrototypeOfIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            setResult(addToGraph(GetPrototypeOf, OpInfo(0), OpInfo(prediction), Edge(get(virtualRegisterForArgument(1, registerOffset)), ObjectUse)));
            return true;
        }

        case IsTypedArrayViewIntrinsic: {
            ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            setResult(addToGraph(IsTypedArrayView, OpInfo(prediction), get(virtualRegisterForArgument(1, registerOffset))));
            return true;
        }

        case StringPrototypeValueOfIntrinsic: {
            insertChecks();
            Node* value = get(virtualRegisterForArgument(0, registerOffset));
            setResult(addToGraph(StringValueOf, value));
            return true;
        }

        case StringPrototypeReplaceIntrinsic: {
            if (argumentCountIncludingThis != 3)
                return false;

            // Don't inline intrinsic if we exited due to "search" not being a RegExp or String object.
            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadType))
                return false;

            // Don't inline intrinsic if we exited due to one of the primordial RegExp checks failing.
            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadCell))
                return false;

            JSGlobalObject* globalObject = m_inlineStackTop->m_codeBlock->globalObject();
            Structure* regExpStructure = globalObject->regExpStructure();
            m_graph.registerStructure(regExpStructure);
            ASSERT(regExpStructure->storedPrototype().isObject());
            ASSERT(regExpStructure->storedPrototype().asCell()->classInfo(*m_vm) == RegExpPrototype::info());

            FrozenValue* regExpPrototypeObjectValue = m_graph.freeze(regExpStructure->storedPrototype());
            Structure* regExpPrototypeStructure = regExpPrototypeObjectValue->structure();

            auto isRegExpPropertySame = [&] (JSValue primordialProperty, UniquedStringImpl* propertyUID) {
                JSValue currentProperty;
                if (!m_graph.getRegExpPrototypeProperty(regExpStructure->storedPrototypeObject(), regExpPrototypeStructure, propertyUID, currentProperty))
                    return false;

                return currentProperty == primordialProperty;
            };

            // Check that searchRegExp.exec is still the primordial RegExp.prototype.exec
            if (!isRegExpPropertySame(globalObject->regExpProtoExecFunction(), m_vm->propertyNames->exec.impl()))
                return false;

            // Check that searchRegExp.global is still the primordial RegExp.prototype.global
            if (!isRegExpPropertySame(globalObject->regExpProtoGlobalGetter(), m_vm->propertyNames->global.impl()))
                return false;

            // Check that searchRegExp.unicode is still the primordial RegExp.prototype.unicode
            if (!isRegExpPropertySame(globalObject->regExpProtoUnicodeGetter(), m_vm->propertyNames->unicode.impl()))
                return false;

            // Check that searchRegExp[Symbol.match] is still the primordial RegExp.prototype[Symbol.replace]
            if (!isRegExpPropertySame(globalObject->regExpProtoSymbolReplaceFunction(), m_vm->propertyNames->replaceSymbol.impl()))
                return false;

            insertChecks();

            Node* resultNode = addToGraph(StringReplace, OpInfo(0), OpInfo(prediction), get(virtualRegisterForArgument(0, registerOffset)), get(virtualRegisterForArgument(1, registerOffset)), get(virtualRegisterForArgument(2, registerOffset)));
            setResult(resultNode);
            return true;
        }
            
        case StringPrototypeReplaceRegExpIntrinsic: {
            if (argumentCountIncludingThis != 3)
                return false;
            
            insertChecks();
            Node* resultNode = addToGraph(StringReplaceRegExp, OpInfo(0), OpInfo(prediction), get(virtualRegisterForArgument(0, registerOffset)), get(virtualRegisterForArgument(1, registerOffset)), get(virtualRegisterForArgument(2, registerOffset)));
            setResult(resultNode);
            return true;
        }
            
        case RoundIntrinsic:
        case FloorIntrinsic:
        case CeilIntrinsic:
        case TruncIntrinsic: {
            if (argumentCountIncludingThis == 1) {
                insertChecks();
                setResult(addToGraph(JSConstant, OpInfo(m_constantNaN)));
                return true;
            }
            insertChecks();
            Node* operand = get(virtualRegisterForArgument(1, registerOffset));
            NodeType op;
            if (intrinsic == RoundIntrinsic)
                op = ArithRound;
            else if (intrinsic == FloorIntrinsic)
                op = ArithFloor;
            else if (intrinsic == CeilIntrinsic)
                op = ArithCeil;
            else {
                ASSERT(intrinsic == TruncIntrinsic);
                op = ArithTrunc;
            }
            Node* roundNode = addToGraph(op, OpInfo(0), OpInfo(prediction), operand);
            setResult(roundNode);
            return true;
        }
        case IMulIntrinsic: {
            if (argumentCountIncludingThis != 3)
                return false;
            insertChecks();
            VirtualRegister leftOperand = virtualRegisterForArgument(1, registerOffset);
            VirtualRegister rightOperand = virtualRegisterForArgument(2, registerOffset);
            Node* left = get(leftOperand);
            Node* right = get(rightOperand);
            setResult(addToGraph(ArithIMul, left, right));
            return true;
        }

        case RandomIntrinsic: {
            if (argumentCountIncludingThis != 1)
                return false;
            insertChecks();
            setResult(addToGraph(ArithRandom));
            return true;
        }
            
        case DFGTrueIntrinsic: {
            insertChecks();
            setResult(jsConstant(jsBoolean(true)));
            return true;
        }

        case FTLTrueIntrinsic: {
            insertChecks();
            setResult(jsConstant(jsBoolean(m_graph.m_plan.isFTL())));
            return true;
        }
            
        case OSRExitIntrinsic: {
            insertChecks();
            addToGraph(ForceOSRExit);
            setResult(addToGraph(JSConstant, OpInfo(m_constantUndefined)));
            return true;
        }
            
        case IsFinalTierIntrinsic: {
            insertChecks();
            setResult(jsConstant(jsBoolean(Options::useFTLJIT() ? m_graph.m_plan.isFTL() : true)));
            return true;
        }
            
        case SetInt32HeapPredictionIntrinsic: {
            insertChecks();
            for (int i = 1; i < argumentCountIncludingThis; ++i) {
                Node* node = get(virtualRegisterForArgument(i, registerOffset));
                if (node->hasHeapPrediction())
                    node->setHeapPrediction(SpecInt32Only);
            }
            setResult(addToGraph(JSConstant, OpInfo(m_constantUndefined)));
            return true;
        }
            
        case CheckInt32Intrinsic: {
            insertChecks();
            for (int i = 1; i < argumentCountIncludingThis; ++i) {
                Node* node = get(virtualRegisterForArgument(i, registerOffset));
                addToGraph(Phantom, Edge(node, Int32Use));
            }
            setResult(jsConstant(jsBoolean(true)));
            return true;
        }
            
        case FiatInt52Intrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;
            insertChecks();
            VirtualRegister operand = virtualRegisterForArgument(1, registerOffset);
            if (enableInt52())
                setResult(addToGraph(FiatInt52, get(operand)));
            else
                setResult(get(operand));
            return true;
        }

        case JSMapGetIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            Node* map = get(virtualRegisterForArgument(0, registerOffset));
            Node* key = get(virtualRegisterForArgument(1, registerOffset));
            Node* normalizedKey = addToGraph(NormalizeMapKey, key);
            Node* hash = addToGraph(MapHash, normalizedKey);
            Node* bucket = addToGraph(GetMapBucket, Edge(map, MapObjectUse), Edge(normalizedKey), Edge(hash));
            Node* resultNode = addToGraph(LoadValueFromMapBucket, OpInfo(BucketOwnerType::Map), OpInfo(prediction), bucket);
            setResult(resultNode);
            return true;
        }

        case JSSetHasIntrinsic:
        case JSMapHasIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            Node* mapOrSet = get(virtualRegisterForArgument(0, registerOffset));
            Node* key = get(virtualRegisterForArgument(1, registerOffset));
            Node* normalizedKey = addToGraph(NormalizeMapKey, key);
            Node* hash = addToGraph(MapHash, normalizedKey);
            UseKind useKind = intrinsic == JSSetHasIntrinsic ? SetObjectUse : MapObjectUse;
            Node* bucket = addToGraph(GetMapBucket, OpInfo(0), Edge(mapOrSet, useKind), Edge(normalizedKey), Edge(hash));
            JSCell* sentinel = nullptr;
            if (intrinsic == JSMapHasIntrinsic)
                sentinel = m_vm->sentinelMapBucket();
            else
                sentinel = m_vm->sentinelSetBucket();

            FrozenValue* frozenPointer = m_graph.freeze(sentinel);
            Node* invertedResult = addToGraph(CompareEqPtr, OpInfo(frozenPointer), bucket);
            Node* resultNode = addToGraph(LogicalNot, invertedResult);
            setResult(resultNode);
            return true;
        }

        case JSSetAddIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            insertChecks();
            Node* base = get(virtualRegisterForArgument(0, registerOffset));
            Node* key = get(virtualRegisterForArgument(1, registerOffset));
            Node* normalizedKey = addToGraph(NormalizeMapKey, key);
            Node* hash = addToGraph(MapHash, normalizedKey);
            addToGraph(SetAdd, base, normalizedKey, hash);
            setResult(base);
            return true;
        }

        case JSMapSetIntrinsic: {
            if (argumentCountIncludingThis != 3)
                return false;

            insertChecks();
            Node* base = get(virtualRegisterForArgument(0, registerOffset));
            Node* key = get(virtualRegisterForArgument(1, registerOffset));
            Node* value = get(virtualRegisterForArgument(2, registerOffset));

            Node* normalizedKey = addToGraph(NormalizeMapKey, key);
            Node* hash = addToGraph(MapHash, normalizedKey);

            addVarArgChild(base);
            addVarArgChild(normalizedKey);
            addVarArgChild(value);
            addVarArgChild(hash);
            addToGraph(Node::VarArg, MapSet, OpInfo(0), OpInfo(0));
            setResult(base);
            return true;
        }

        case JSSetBucketHeadIntrinsic:
        case JSMapBucketHeadIntrinsic: {
            ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            Node* map = get(virtualRegisterForArgument(1, registerOffset));
            UseKind useKind = intrinsic == JSSetBucketHeadIntrinsic ? SetObjectUse : MapObjectUse;
            Node* resultNode = addToGraph(GetMapBucketHead, Edge(map, useKind));
            setResult(resultNode);
            return true;
        }

        case JSSetBucketNextIntrinsic:
        case JSMapBucketNextIntrinsic: {
            ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
            BucketOwnerType type = intrinsic == JSSetBucketNextIntrinsic ? BucketOwnerType::Set : BucketOwnerType::Map;
            Node* resultNode = addToGraph(GetMapBucketNext, OpInfo(type), bucket);
            setResult(resultNode);
            return true;
        }

        case JSSetBucketKeyIntrinsic:
        case JSMapBucketKeyIntrinsic: {
            ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
            BucketOwnerType type = intrinsic == JSSetBucketKeyIntrinsic ? BucketOwnerType::Set : BucketOwnerType::Map;
            Node* resultNode = addToGraph(LoadKeyFromMapBucket, OpInfo(type), OpInfo(prediction), bucket);
            setResult(resultNode);
            return true;
        }

        case JSMapBucketValueIntrinsic: {
            ASSERT(argumentCountIncludingThis == 2);

            insertChecks();
            Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
            Node* resultNode = addToGraph(LoadValueFromMapBucket, OpInfo(BucketOwnerType::Map), OpInfo(prediction), bucket);
            setResult(resultNode);
            return true;
        }

        case JSWeakMapGetIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadType))
                return false;

            insertChecks();
            Node* map = get(virtualRegisterForArgument(0, registerOffset));
            Node* key = get(virtualRegisterForArgument(1, registerOffset));
            addToGraph(Check, Edge(key, ObjectUse));
            Node* hash = addToGraph(MapHash, key);
            Node* holder = addToGraph(WeakMapGet, Edge(map, WeakMapObjectUse), Edge(key, ObjectUse), Edge(hash, Int32Use));
            Node* resultNode = addToGraph(ExtractValueFromWeakMapGet, OpInfo(), OpInfo(prediction), holder);

            setResult(resultNode);
            return true;
        }

        case JSWeakMapHasIntrinsic: {
            if (argumentCountIncludingThis != 2)
                return false;

            if (m_inlineStackTop->m_exitProfile.hasExitSite(m_currentIndex, BadType))