Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp

   1 /*
   2  * Copyright (C) 2011-2017 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "DFGSpeculativeJIT.h"
  28
  29 #if ENABLE(DFG_JIT)
  30
  31 #include "BinarySwitch.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGArrayifySlowPathGenerator.h"
  34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
  35 #include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
  36 #include "DFGCapabilities.h"
  37 #include "DFGMayExit.h"
  38 #include "DFGOSRExitFuzz.h"
  39 #include "DFGSaneStringGetByValSlowPathGenerator.h"
  40 #include "DFGSlowPathGenerator.h"
  41 #include "DFGSnippetParams.h"
  42 #include "DirectArguments.h"
  43 #include "JITAddGenerator.h"
  44 #include "JITBitAndGenerator.h"
  45 #include "JITBitOrGenerator.h"
  46 #include "JITBitXorGenerator.h"
  47 #include "JITDivGenerator.h"
  48 #include "JITLeftShiftGenerator.h"
  49 #include "JITMulGenerator.h"
  50 #include "JITRightShiftGenerator.h"
  51 #include "JITSubGenerator.h"
  52 #include "JSAsyncFunction.h"
  53 #include "JSAsyncGeneratorFunction.h"
  54 #include "JSCInlines.h"
  55 #include "JSFixedArray.h"
  56 #include "JSGeneratorFunction.h"
  57 #include "JSLexicalEnvironment.h"
  58 #include "JSPropertyNameEnumerator.h"
  59 #include "LinkBuffer.h"
  60 #include "RegExpConstructor.h"
  61 #include "ScopedArguments.h"
  62 #include "ScratchRegisterAllocator.h"
  63 #include "SuperSampler.h"
  64 #include "WeakMapImpl.h"
  65 #include <wtf/BitVector.h>
  66 #include <wtf/Box.h>
  67 #include <wtf/MathExtras.h>
  68
  69 namespace JSC { namespace DFG {
  70
  71 SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
  72     : m_compileOkay(true)
  73     , m_jit(jit)
  74     , m_currentNode(0)
  75     , m_lastGeneratedNode(LastNodeType)
  76     , m_indexInBlock(0)
  77     , m_generationInfo(m_jit.graph().frameRegisterCount())
  78     , m_state(m_jit.graph())
  79     , m_interpreter(m_jit.graph(), m_state)
  80     , m_stream(&jit.jitCode()->variableEventStream)
  81     , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
  82 {
  83 }
  84
  85 SpeculativeJIT::~SpeculativeJIT()
  86 {
  87 }
  88
  89 void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
  90 {
  91     IndexingType indexingType = structure->indexingType();
  92     bool hasIndexingHeader = hasIndexedProperties(indexingType);
  93
  94     unsigned inlineCapacity = structure->inlineCapacity();
  95     unsigned outOfLineCapacity = structure->outOfLineCapacity();
  96
  97     GPRTemporary scratch(this);
  98     GPRTemporary scratch2(this);
  99     GPRReg scratchGPR = scratch.gpr();
 100     GPRReg scratch2GPR = scratch2.gpr();
 101
 102     ASSERT(vectorLength >= numElements);
 103     vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
 104
 105     JITCompiler::JumpList slowCases;
 106
 107     size_t size = 0;
 108     if (hasIndexingHeader)
 109         size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
 110     size += outOfLineCapacity * sizeof(JSValue);
 111
 112     m_jit.move(TrustedImmPtr(0), storageGPR);
 113
 114     if (size) {
 115         if (MarkedAllocator* allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
 116             m_jit.move(TrustedImmPtr(allocator), scratchGPR);
 117             m_jit.emitAllocate(storageGPR, allocator, scratchGPR, scratch2GPR, slowCases);
 118
 119             m_jit.addPtr(
 120                 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
 121                 storageGPR);
 122
 123             if (hasIndexingHeader)
 124                 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
 125         } else
 126             slowCases.append(m_jit.jump());
 127     }
 128
 129     size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
 130     MarkedAllocator* allocatorPtr = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorForNonVirtual(allocationSize, AllocatorForMode::AllocatorIfExists);
 131     if (allocatorPtr) {
 132         m_jit.move(TrustedImmPtr(allocatorPtr), scratchGPR);
 133         uint32_t mask = Butterfly::computeIndexingMaskForVectorLength(vectorLength);
 134         emitAllocateJSObject(resultGPR, allocatorPtr, scratchGPR, TrustedImmPtr(structure), storageGPR, TrustedImm32(mask), scratch2GPR, slowCases);
 135         m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
 136     } else
 137         slowCases.append(m_jit.jump());
 138
 139     // I want a slow path that also loads out the storage pointer, and that's
 140     // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
 141     // of work for a very small piece of functionality. :-/
 142     addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
 143         slowCases, this, operationNewRawObject, resultGPR, storageGPR,
 144         structure, vectorLength));
 145
 146     if (numElements < vectorLength && LIKELY(!hasUndecided(structure->indexingType()))) {
 147 #if USE(JSVALUE64)
 148         if (hasDouble(structure->indexingType()))
 149             m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
 150         else
 151             m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
 152         for (unsigned i = numElements; i < vectorLength; ++i)
 153             m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
 154 #else
 155         EncodedValueDescriptor value;
 156         if (hasDouble(structure->indexingType()))
 157             value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
 158         else
 159             value.asInt64 = JSValue::encode(JSValue());
 160         for (unsigned i = numElements; i < vectorLength; ++i) {
 161             m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
 162             m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
 163         }
 164 #endif
 165     }
 166
 167     if (hasIndexingHeader)
 168         m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
 169
 170     m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
 171
 172     m_jit.mutatorFence(*m_jit.vm());
 173 }
 174
 175 void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
 176 {
 177     if (inlineCallFrame && !inlineCallFrame->isVarargs())
 178         m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
 179     else {
 180         VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
 181         m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
 182         if (!includeThis)
 183             m_jit.sub32(TrustedImm32(1), lengthGPR);
 184     }
 185 }
 186
 187 void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
 188 {
 189     emitGetLength(origin.inlineCallFrame, lengthGPR, includeThis);
 190 }
 191
 192 void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
 193 {
 194     if (origin.inlineCallFrame) {
 195         if (origin.inlineCallFrame->isClosureCall) {
 196             m_jit.loadPtr(
 197                 JITCompiler::addressFor(origin.inlineCallFrame->calleeRecovery.virtualRegister()),
 198                 calleeGPR);
 199         } else {
 200             m_jit.move(
 201                 TrustedImmPtr::weakPointer(m_jit.graph(), origin.inlineCallFrame->calleeRecovery.constant().asCell()),
 202                 calleeGPR);
 203         }
 204     } else
 205         m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
 206 }
 207
 208 void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
 209 {
 210     m_jit.addPtr(
 211         TrustedImm32(
 212             JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
 213         GPRInfo::callFrameRegister, startGPR);
 214 }
 215
 216 MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
 217 {
 218     if (!Options::useOSRExitFuzz()
 219         || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
 220         || !doOSRExitFuzzing())
 221         return MacroAssembler::Jump();
 222
 223     MacroAssembler::Jump result;
 224
 225     m_jit.pushToSave(GPRInfo::regT0);
 226     m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
 227     m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
 228     m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
 229     unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
 230     unsigned at = Options::fireOSRExitFuzzAt();
 231     if (at || atOrAfter) {
 232         unsigned threshold;
 233         MacroAssembler::RelationalCondition condition;
 234         if (atOrAfter) {
 235             threshold = atOrAfter;
 236             condition = MacroAssembler::Below;
 237         } else {
 238             threshold = at;
 239             condition = MacroAssembler::NotEqual;
 240         }
 241         MacroAssembler::Jump ok = m_jit.branch32(
 242             condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
 243         m_jit.popToRestore(GPRInfo::regT0);
 244         result = m_jit.jump();
 245         ok.link(&m_jit);
 246     }
 247     m_jit.popToRestore(GPRInfo::regT0);
 248
 249     return result;
 250 }
 251
 252 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
 253 {
 254     if (!m_compileOkay)
 255         return;
 256     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
 257     if (fuzzJump.isSet()) {
 258         JITCompiler::JumpList jumpsToFail;
 259         jumpsToFail.append(fuzzJump);
 260         jumpsToFail.append(jumpToFail);
 261         m_jit.appendExitInfo(jumpsToFail);
 262     } else
 263         m_jit.appendExitInfo(jumpToFail);
 264     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
 265 }
 266
 267 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
 268 {
 269     if (!m_compileOkay)
 270         return;
 271     JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
 272     if (fuzzJump.isSet()) {
 273         JITCompiler::JumpList myJumpsToFail;
 274         myJumpsToFail.append(jumpsToFail);
 275         myJumpsToFail.append(fuzzJump);
 276         m_jit.appendExitInfo(myJumpsToFail);
 277     } else
 278         m_jit.appendExitInfo(jumpsToFail);
 279     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
 280 }
 281
 282 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
 283 {
 284     if (!m_compileOkay)
 285         return OSRExitJumpPlaceholder();
 286     unsigned index = m_jit.jitCode()->osrExit.size();
 287     m_jit.appendExitInfo();
 288     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
 289     return OSRExitJumpPlaceholder(index);
 290 }
 291
 292 OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
 293 {
 294     return speculationCheck(kind, jsValueSource, nodeUse.node());
 295 }
 296
 297 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
 298 {
 299     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
 300 }
 301
 302 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
 303 {
 304     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
 305 }
 306
 307 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
 308 {
 309     if (!m_compileOkay)
 310         return;
 311     unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
 312     m_jit.appendExitInfo(jumpToFail);
 313     m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
 314 }
 315
 316 void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
 317 {
 318     speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
 319 }
 320
 321 void SpeculativeJIT::emitInvalidationPoint(Node* node)
 322 {
 323     if (!m_compileOkay)
 324         return;
 325     OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
 326     m_jit.jitCode()->appendOSRExit(OSRExit(
 327         UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
 328         this, m_stream->size()));
 329     info.m_replacementSource = m_jit.watchpointLabel();
 330     ASSERT(info.m_replacementSource.isSet());
 331     noResult(node);
 332 }
 333
 334 void SpeculativeJIT::unreachable(Node* node)
 335 {
 336     m_compileOkay = false;
 337     m_jit.abortWithReason(DFGUnreachableNode, node->op());
 338 }
 339
 340 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
 341 {
 342     if (!m_compileOkay)
 343         return;
 344     speculationCheck(kind, jsValueRegs, node, m_jit.jump());
 345     m_compileOkay = false;
 346     if (verboseCompilationEnabled())
 347         dataLog("Bailing compilation.\n");
 348 }
 349
 350 void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
 351 {
 352     terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
 353 }
 354
 355 void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
 356 {
 357     ASSERT(needsTypeCheck(edge, typesPassedThrough));
 358     m_interpreter.filter(edge, typesPassedThrough);
 359     speculationCheck(exitKind, source, edge.node(), jumpToFail);
 360 }
 361
 362 RegisterSet SpeculativeJIT::usedRegisters()
 363 {
 364     RegisterSet result;
 365
 366     for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
 367         GPRReg gpr = GPRInfo::toRegister(i);
 368         if (m_gprs.isInUse(gpr))
 369             result.set(gpr);
 370     }
 371     for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
 372         FPRReg fpr = FPRInfo::toRegister(i);
 373         if (m_fprs.isInUse(fpr))
 374             result.set(fpr);
 375     }
 376
 377     result.merge(RegisterSet::stubUnavailableRegisters());
 378
 379     return result;
 380 }
 381
 382 void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
 383 {
 384     m_slowPathGenerators.append(WTFMove(slowPathGenerator));
 385 }
 386
 387 void SpeculativeJIT::addSlowPathGenerator(std::function<void()> lambda)
 388 {
 389     m_slowPathLambdas.append(SlowPathLambda{ lambda, m_currentNode, static_cast<unsigned>(m_stream->size()) });
 390 }
 391
 392 void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
 393 {
 394     for (auto& slowPathGenerator : m_slowPathGenerators) {
 395         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
 396         slowPathGenerator->generate(this);
 397     }
 398     for (auto& slowPathLambda : m_slowPathLambdas) {
 399         Node* currentNode = slowPathLambda.currentNode;
 400         m_currentNode = currentNode;
 401         m_outOfLineStreamIndex = slowPathLambda.streamIndex;
 402         pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
 403         slowPathLambda.generator();
 404         m_outOfLineStreamIndex = std::nullopt;
 405     }
 406 }
 407
 408 void SpeculativeJIT::clearGenerationInfo()
 409 {
 410     for (unsigned i = 0; i < m_generationInfo.size(); ++i)
 411         m_generationInfo[i] = GenerationInfo();
 412     m_gprs = RegisterBank<GPRInfo>();
 413     m_fprs = RegisterBank<FPRInfo>();
 414 }
 415
 416 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
 417 {
 418     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
 419     Node* node = info.node();
 420     DataFormat registerFormat = info.registerFormat();
 421     ASSERT(registerFormat != DataFormatNone);
 422     ASSERT(registerFormat != DataFormatDouble);
 423
 424     SilentSpillAction spillAction;
 425     SilentFillAction fillAction;
 426
 427     if (!info.needsSpill())
 428         spillAction = DoNothingForSpill;
 429     else {
 430 #if USE(JSVALUE64)
 431         ASSERT(info.gpr() == source);
 432         if (registerFormat == DataFormatInt32)
 433             spillAction = Store32Payload;
 434         else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
 435             spillAction = StorePtr;
 436         else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
 437             spillAction = Store64;
 438         else {
 439             ASSERT(registerFormat & DataFormatJS);
 440             spillAction = Store64;
 441         }
 442 #elif USE(JSVALUE32_64)
 443         if (registerFormat & DataFormatJS) {
 444             ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
 445             spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
 446         } else {
 447             ASSERT(info.gpr() == source);
 448             spillAction = Store32Payload;
 449         }
 450 #endif
 451     }
 452
 453     if (registerFormat == DataFormatInt32) {
 454         ASSERT(info.gpr() == source);
 455         ASSERT(isJSInt32(info.registerFormat()));
 456         if (node->hasConstant()) {
 457             ASSERT(node->isInt32Constant());
 458             fillAction = SetInt32Constant;
 459         } else
 460             fillAction = Load32Payload;
 461     } else if (registerFormat == DataFormatBoolean) {
 462 #if USE(JSVALUE64)
 463         RELEASE_ASSERT_NOT_REACHED();
 464 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
 465         fillAction = DoNothingForFill;
 466 #endif
 467 #elif USE(JSVALUE32_64)
 468         ASSERT(info.gpr() == source);
 469         if (node->hasConstant()) {
 470             ASSERT(node->isBooleanConstant());
 471             fillAction = SetBooleanConstant;
 472         } else
 473             fillAction = Load32Payload;
 474 #endif
 475     } else if (registerFormat == DataFormatCell) {
 476         ASSERT(info.gpr() == source);
 477         if (node->hasConstant()) {
 478             DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
 479             node->asCell(); // To get the assertion.
 480             fillAction = SetCellConstant;
 481         } else {
 482 #if USE(JSVALUE64)
 483             fillAction = LoadPtr;
 484 #else
 485             fillAction = Load32Payload;
 486 #endif
 487         }
 488     } else if (registerFormat == DataFormatStorage) {
 489         ASSERT(info.gpr() == source);
 490         fillAction = LoadPtr;
 491     } else if (registerFormat == DataFormatInt52) {
 492         if (node->hasConstant())
 493             fillAction = SetInt52Constant;
 494         else if (info.spillFormat() == DataFormatInt52)
 495             fillAction = Load64;
 496         else if (info.spillFormat() == DataFormatStrictInt52)
 497             fillAction = Load64ShiftInt52Left;
 498         else if (info.spillFormat() == DataFormatNone)
 499             fillAction = Load64;
 500         else {
 501             RELEASE_ASSERT_NOT_REACHED();
 502 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
 503             fillAction = Load64; // Make GCC happy.
 504 #endif
 505         }
 506     } else if (registerFormat == DataFormatStrictInt52) {
 507         if (node->hasConstant())
 508             fillAction = SetStrictInt52Constant;
 509         else if (info.spillFormat() == DataFormatInt52)
 510             fillAction = Load64ShiftInt52Right;
 511         else if (info.spillFormat() == DataFormatStrictInt52)
 512             fillAction = Load64;
 513         else if (info.spillFormat() == DataFormatNone)
 514             fillAction = Load64;
 515         else {
 516             RELEASE_ASSERT_NOT_REACHED();
 517 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
 518             fillAction = Load64; // Make GCC happy.
 519 #endif
 520         }
 521     } else {
 522         ASSERT(registerFormat & DataFormatJS);
 523 #if USE(JSVALUE64)
 524         ASSERT(info.gpr() == source);
 525         if (node->hasConstant()) {
 526             if (node->isCellConstant())
 527                 fillAction = SetTrustedJSConstant;
 528             else
 529                 fillAction = SetJSConstant;
 530         } else if (info.spillFormat() == DataFormatInt32) {
 531             ASSERT(registerFormat == DataFormatJSInt32);
 532             fillAction = Load32PayloadBoxInt;
 533         } else
 534             fillAction = Load64;
 535 #else
 536         ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
 537         if (node->hasConstant())
 538             fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
 539         else if (info.payloadGPR() == source)
 540             fillAction = Load32Payload;
 541         else { // Fill the Tag
 542             switch (info.spillFormat()) {
 543             case DataFormatInt32:
 544                 ASSERT(registerFormat == DataFormatJSInt32);
 545                 fillAction = SetInt32Tag;
 546                 break;
 547             case DataFormatCell:
 548                 ASSERT(registerFormat == DataFormatJSCell);
 549                 fillAction = SetCellTag;
 550                 break;
 551             case DataFormatBoolean:
 552                 ASSERT(registerFormat == DataFormatJSBoolean);
 553                 fillAction = SetBooleanTag;
 554                 break;
 555             default:
 556                 fillAction = Load32Tag;
 557                 break;
 558             }
 559         }
 560 #endif
 561     }
 562
 563     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
 564 }
 565
 566 SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
 567 {
 568     GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
 569     Node* node = info.node();
 570     ASSERT(info.registerFormat() == DataFormatDouble);
 571
 572     SilentSpillAction spillAction;
 573     SilentFillAction fillAction;
 574
 575     if (!info.needsSpill())
 576         spillAction = DoNothingForSpill;
 577     else {
 578         ASSERT(!node->hasConstant());
 579         ASSERT(info.spillFormat() == DataFormatNone);
 580         ASSERT(info.fpr() == source);
 581         spillAction = StoreDouble;
 582     }
 583
 584 #if USE(JSVALUE64)
 585     if (node->hasConstant()) {
 586         node->asNumber(); // To get the assertion.
 587         fillAction = SetDoubleConstant;
 588     } else {
 589         ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
 590         fillAction = LoadDouble;
 591     }
 592 #elif USE(JSVALUE32_64)
 593     ASSERT(info.registerFormat() == DataFormatDouble);
 594     if (node->hasConstant()) {
 595         node->asNumber(); // To get the assertion.
 596         fillAction = SetDoubleConstant;
 597     } else
 598         fillAction = LoadDouble;
 599 #endif
 600
 601     return SilentRegisterSavePlan(spillAction, fillAction, node, source);
 602 }
 603
 604 void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
 605 {
 606     switch (plan.spillAction()) {
 607     case DoNothingForSpill:
 608         break;
 609     case Store32Tag:
 610         m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
 611         break;
 612     case Store32Payload:
 613         m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
 614         break;
 615     case StorePtr:
 616         m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
 617         break;
 618 #if USE(JSVALUE64)
 619     case Store64:
 620         m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
 621         break;
 622 #endif
 623     case StoreDouble:
 624         m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
 625         break;
 626     default:
 627         RELEASE_ASSERT_NOT_REACHED();
 628     }
 629 }
 630
 631 void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
 632 {
 633     switch (plan.fillAction()) {
 634     case DoNothingForFill:
 635         break;
 636     case SetInt32Constant:
 637         m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
 638         break;
 639 #if USE(JSVALUE64)
 640     case SetInt52Constant:
 641         m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
 642         break;
 643     case SetStrictInt52Constant:
 644         m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
 645         break;
 646 #endif // USE(JSVALUE64)
 647     case SetBooleanConstant:
 648         m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
 649         break;
 650     case SetCellConstant:
 651         ASSERT(plan.node()->constant()->value().isCell());
 652         m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
 653         break;
 654 #if USE(JSVALUE64)
 655     case SetTrustedJSConstant:
 656         m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
 657         break;
 658     case SetJSConstant:
 659         m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
 660         break;
 661     case SetDoubleConstant:
 662         m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
 663         break;
 664     case Load32PayloadBoxInt:
 665         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
 666         m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
 667         break;
 668     case Load32PayloadConvertToInt52:
 669         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
 670         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
 671         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
 672         break;
 673     case Load32PayloadSignExtend:
 674         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
 675         m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
 676         break;
 677 #else
 678     case SetJSConstantTag:
 679         m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
 680         break;
 681     case SetJSConstantPayload:
 682         m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
 683         break;
 684     case SetInt32Tag:
 685         m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
 686         break;
 687     case SetCellTag:
 688         m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
 689         break;
 690     case SetBooleanTag:
 691         m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
 692         break;
 693     case SetDoubleConstant:
 694         m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
 695         break;
 696 #endif
 697     case Load32Tag:
 698         m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
 699         break;
 700     case Load32Payload:
 701         m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
 702         break;
 703     case LoadPtr:
 704         m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
 705         break;
 706 #if USE(JSVALUE64)
 707     case Load64:
 708         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
 709         break;
 710     case Load64ShiftInt52Right:
 711         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
 712         m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
 713         break;
 714     case Load64ShiftInt52Left:
 715         m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
 716         m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
 717         break;
 718 #endif
 719     case LoadDouble:
 720         m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
 721         break;
 722     default:
 723         RELEASE_ASSERT_NOT_REACHED();
 724     }
 725 }
 726
 727 JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, IndexingType shape)
 728 {
 729     switch (arrayMode.arrayClass()) {
 730     case Array::OriginalArray: {
 731         CRASH();
 732 #if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
 733         JITCompiler::Jump result; // I already know that VC++ takes unkindly to the expression "return Jump()", so I'm doing it this way in anticipation of someone eventually using VC++ to compile the DFG.
 734         return result;
 735 #endif
 736     }
 737
 738     case Array::Array:
 739         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
 740         return m_jit.branch32(
 741             MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape));
 742
 743     case Array::NonArray:
 744     case Array::OriginalNonArray:
 745         m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
 746         return m_jit.branch32(
 747             MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
 748
 749     case Array::PossiblyArray:
 750         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
 751         return m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape));
 752     }
 753
 754     RELEASE_ASSERT_NOT_REACHED();
 755     return JITCompiler::Jump();
 756 }
 757
 758 JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
 759 {
 760     JITCompiler::JumpList result;
 761
 762     switch (arrayMode.type()) {
 763     case Array::Int32:
 764     case Array::Double:
 765     case Array::Contiguous:
 766     case Array::Undecided:
 767         return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, arrayMode.shapeMask());
 768
 769     case Array::ArrayStorage:
 770     case Array::SlowPutArrayStorage: {
 771         ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
 772
 773         if (arrayMode.isJSArray()) {
 774             if (arrayMode.isSlowPut()) {
 775                 result.append(
 776                     m_jit.branchTest32(
 777                         MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
 778                 m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
 779                 m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
 780                 result.append(
 781                     m_jit.branch32(
 782                         MacroAssembler::Above, tempGPR,
 783                         TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
 784                 break;
 785             }
 786             m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR);
 787             result.append(
 788                 m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | ArrayStorageShape)));
 789             break;
 790         }
 791         m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
 792         if (arrayMode.isSlowPut()) {
 793             m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
 794             result.append(
 795                 m_jit.branch32(
 796                     MacroAssembler::Above, tempGPR,
 797                     TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
 798             break;
 799         }
 800         result.append(
 801             m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape)));
 802         break;
 803     }
 804     default:
 805         CRASH();
 806         break;
 807     }
 808
 809     return result;
 810 }
 811
 812 void SpeculativeJIT::checkArray(Node* node)
 813 {
 814     ASSERT(node->arrayMode().isSpecific());
 815     ASSERT(!node->arrayMode().doesConversion());
 816
 817     SpeculateCellOperand base(this, node->child1());
 818     GPRReg baseReg = base.gpr();
 819
 820     if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
 821         noResult(m_currentNode);
 822         return;
 823     }
 824
 825     const ClassInfo* expectedClassInfo = 0;
 826
 827     switch (node->arrayMode().type()) {
 828     case Array::AnyTypedArray:
 829     case Array::String:
 830         RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
 831         break;
 832     case Array::Int32:
 833     case Array::Double:
 834     case Array::Contiguous:
 835     case Array::Undecided:
 836     case Array::ArrayStorage:
 837     case Array::SlowPutArrayStorage: {
 838         GPRTemporary temp(this);
 839         GPRReg tempGPR = temp.gpr();
 840         m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
 841         speculationCheck(
 842             BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
 843             jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
 844
 845         noResult(m_currentNode);
 846         return;
 847     }
 848     case Array::DirectArguments:
 849         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
 850         noResult(m_currentNode);
 851         return;
 852     case Array::ScopedArguments:
 853         speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
 854         noResult(m_currentNode);
 855         return;
 856     default:
 857         speculateCellTypeWithoutTypeFiltering(
 858             node->child1(), baseReg,
 859             typeForTypedArrayType(node->arrayMode().typedArrayType()));
 860         noResult(m_currentNode);
 861         return;
 862     }
 863
 864     RELEASE_ASSERT(expectedClassInfo);
 865
 866     GPRTemporary temp(this);
 867     GPRTemporary temp2(this);
 868     m_jit.emitLoadStructure(*m_jit.vm(), baseReg, temp.gpr(), temp2.gpr());
 869     speculationCheck(
 870         BadType, JSValueSource::unboxedCell(baseReg), node,
 871         m_jit.branchPtr(
 872             MacroAssembler::NotEqual,
 873             MacroAssembler::Address(temp.gpr(), Structure::classInfoOffset()),
 874             TrustedImmPtr(PoisonedClassInfoPtr(expectedClassInfo).bits())));
 875
 876     noResult(m_currentNode);
 877 }
 878
 879 void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
 880 {
 881     ASSERT(node->arrayMode().doesConversion());
 882
 883     GPRTemporary temp(this);
 884     GPRTemporary structure;
 885     GPRReg tempGPR = temp.gpr();
 886     GPRReg structureGPR = InvalidGPRReg;
 887
 888     if (node->op() != ArrayifyToStructure) {
 889         GPRTemporary realStructure(this);
 890         structure.adopt(realStructure);
 891         structureGPR = structure.gpr();
 892     }
 893
 894     // We can skip all that comes next if we already have array storage.
 895     MacroAssembler::JumpList slowPath;
 896
 897     if (node->op() == ArrayifyToStructure) {
 898         slowPath.append(m_jit.branchWeakStructure(
 899             JITCompiler::NotEqual,
 900             JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
 901             node->structure()));
 902     } else {
 903         m_jit.load8(
 904             MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
 905
 906         slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
 907     }
 908
 909     addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
 910         slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
 911
 912     noResult(m_currentNode);
 913 }
 914
 915 void SpeculativeJIT::arrayify(Node* node)
 916 {
 917     ASSERT(node->arrayMode().isSpecific());
 918
 919     SpeculateCellOperand base(this, node->child1());
 920
 921     if (!node->child2()) {
 922         arrayify(node, base.gpr(), InvalidGPRReg);
 923         return;
 924     }
 925
 926     SpeculateInt32Operand property(this, node->child2());
 927
 928     arrayify(node, base.gpr(), property.gpr());
 929 }
 930
 931 GPRReg SpeculativeJIT::fillStorage(Edge edge)
 932 {
 933     VirtualRegister virtualRegister = edge->virtualRegister();
 934     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 935
 936     switch (info.registerFormat()) {
 937     case DataFormatNone: {
 938         if (info.spillFormat() == DataFormatStorage) {
 939             GPRReg gpr = allocate();
 940             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 941             m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
 942             info.fillStorage(*m_stream, gpr);
 943             return gpr;
 944         }
 945
 946         // Must be a cell; fill it as a cell and then return the pointer.
 947         return fillSpeculateCell(edge);
 948     }
 949
 950     case DataFormatStorage: {
 951         GPRReg gpr = info.gpr();
 952         m_gprs.lock(gpr);
 953         return gpr;
 954     }
 955
 956     default:
 957         return fillSpeculateCell(edge);
 958     }
 959 }
 960
 961 void SpeculativeJIT::useChildren(Node* node)
 962 {
 963     if (node->flags() & NodeHasVarArgs) {
 964         for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
 965             if (!!m_jit.graph().m_varArgChildren[childIdx])
 966                 use(m_jit.graph().m_varArgChildren[childIdx]);
 967         }
 968     } else {
 969         Edge child1 = node->child1();
 970         if (!child1) {
 971             ASSERT(!node->child2() && !node->child3());
 972             return;
 973         }
 974         use(child1);
 975
 976         Edge child2 = node->child2();
 977         if (!child2) {
 978             ASSERT(!node->child3());
 979             return;
 980         }
 981         use(child2);
 982
 983         Edge child3 = node->child3();
 984         if (!child3)
 985             return;
 986         use(child3);
 987     }
 988 }
 989
 990 void SpeculativeJIT::compileTryGetById(Node* node)
 991 {
 992     switch (node->child1().useKind()) {
 993     case CellUse: {
 994         SpeculateCellOperand base(this, node->child1());
 995         JSValueRegsTemporary result(this, Reuse, base);
 996
 997         JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
 998         JSValueRegs resultRegs = result.regs();
 999
1000         base.use();
1001
1002         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, AccessType::TryGet);
1003
1004         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1005         break;
1006     }
1007
1008     case UntypedUse: {
1009         JSValueOperand base(this, node->child1());
1010         JSValueRegsTemporary result(this, Reuse, base);
1011
1012         JSValueRegs baseRegs = base.jsValueRegs();
1013         JSValueRegs resultRegs = result.regs();
1014
1015         base.use();
1016
1017         JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1018
1019         cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, AccessType::TryGet);
1020
1021         jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1022         break;
1023     }
1024
1025     default:
1026         DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1027         break;
1028     }
1029 }
1030
1031 void SpeculativeJIT::compileIn(Node* node)
1032 {
1033     SpeculateCellOperand base(this, node->child1());
1034     GPRReg baseGPR = base.gpr();
1035
1036     if (JSString* string = node->child2()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
1037         if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
1038             StructureStubInfo* stubInfo = m_jit.codeBlock()->addStubInfo(AccessType::In);
1039
1040             GPRTemporary result(this);
1041             GPRReg resultGPR = result.gpr();
1042
1043             use(node->child2());
1044
1045             MacroAssembler::PatchableJump jump = m_jit.patchableJump();
1046             MacroAssembler::Label done = m_jit.label();
1047
1048             // Since this block is executed only when the result of string->tryGetValueImpl() is atomic,
1049             // we can cast it to const AtomicStringImpl* safely.
1050             auto slowPath = slowPathCall(
1051                 jump.m_jump, this, operationInOptimize,
1052                 JSValueRegs::payloadOnly(resultGPR), stubInfo, baseGPR,
1053                 static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()));
1054
1055             stubInfo->callSiteIndex = m_jit.addCallSite(node->origin.semantic);
1056             stubInfo->codeOrigin = node->origin.semantic;
1057             stubInfo->patch.baseGPR = static_cast<int8_t>(baseGPR);
1058             stubInfo->patch.valueGPR = static_cast<int8_t>(resultGPR);
1059             stubInfo->patch.thisGPR = static_cast<int8_t>(InvalidGPRReg);
1060 #if USE(JSVALUE32_64)
1061             stubInfo->patch.valueTagGPR = static_cast<int8_t>(InvalidGPRReg);
1062             stubInfo->patch.baseTagGPR = static_cast<int8_t>(InvalidGPRReg);
1063             stubInfo->patch.thisTagGPR = static_cast<int8_t>(InvalidGPRReg);
1064 #endif
1065             stubInfo->patch.usedRegisters = usedRegisters();
1066
1067             m_jit.addIn(InRecord(jump, done, slowPath.get(), stubInfo));
1068             addSlowPathGenerator(WTFMove(slowPath));
1069
1070             base.use();
1071
1072             blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1073             return;
1074         }
1075     }
1076
1077     JSValueOperand key(this, node->child2());
1078     JSValueRegs regs = key.jsValueRegs();
1079
1080     GPRFlushedCallResult result(this);
1081     GPRReg resultGPR = result.gpr();
1082
1083     base.use();
1084     key.use();
1085
1086     flushRegisters();
1087     callOperation(
1088         operationGenericIn, extractResult(JSValueRegs::payloadOnly(resultGPR)),
1089         baseGPR, regs);
1090     m_jit.exceptionCheck();
1091     blessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1092 }
1093
1094 void SpeculativeJIT::compileDeleteById(Node* node)
1095 {
1096     JSValueOperand value(this, node->child1());
1097     GPRFlushedCallResult result(this);
1098
1099     JSValueRegs valueRegs = value.jsValueRegs();
1100     GPRReg resultGPR = result.gpr();
1101
1102     value.use();
1103
1104     flushRegisters();
1105     callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1106     m_jit.exceptionCheck();
1107
1108     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1109 }
1110
1111 void SpeculativeJIT::compileDeleteByVal(Node* node)
1112 {
1113     JSValueOperand base(this, node->child1());
1114     JSValueOperand key(this, node->child2());
1115     GPRFlushedCallResult result(this);
1116
1117     JSValueRegs baseRegs = base.jsValueRegs();
1118     JSValueRegs keyRegs = key.jsValueRegs();
1119     GPRReg resultGPR = result.gpr();
1120
1121     base.use();
1122     key.use();
1123
1124     flushRegisters();
1125     callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1126     m_jit.exceptionCheck();
1127
1128     unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1129 }
1130
1131 void SpeculativeJIT::compilePushWithScope(Node* node)
1132 {
1133     SpeculateCellOperand currentScope(this, node->child1());
1134     GPRReg currentScopeGPR = currentScope.gpr();
1135
1136     GPRFlushedCallResult result(this);
1137     GPRReg resultGPR = result.gpr();
1138
1139     auto objectEdge = node->child2();
1140     if (objectEdge.useKind() == ObjectUse) {
1141         SpeculateCellOperand object(this, objectEdge);
1142         GPRReg objectGPR = object.gpr();
1143         speculateObject(objectEdge, objectGPR);
1144
1145         flushRegisters();
1146         callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1147         // No exception check here as we did not have to call toObject().
1148     } else {
1149         ASSERT(objectEdge.useKind() == UntypedUse);
1150         JSValueOperand object(this, objectEdge);
1151         JSValueRegs objectRegs = object.jsValueRegs();
1152
1153         flushRegisters();
1154         callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1155         m_jit.exceptionCheck();
1156     }
1157
1158     cellResult(resultGPR, node);
1159 }
1160
1161 bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
1162 {
1163     unsigned branchIndexInBlock = detectPeepHoleBranch();
1164     if (branchIndexInBlock != UINT_MAX) {
1165         Node* branchNode = m_block->at(branchIndexInBlock);
1166
1167         ASSERT(node->adjustedRefCount() == 1);
1168
1169         nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction);
1170
1171         m_indexInBlock = branchIndexInBlock;
1172         m_currentNode = branchNode;
1173
1174         return true;
1175     }
1176
1177     nonSpeculativeNonPeepholeCompare(node, cond, helperFunction);
1178
1179     return false;
1180 }
1181
1182 bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1183 {
1184     unsigned branchIndexInBlock = detectPeepHoleBranch();
1185     if (branchIndexInBlock != UINT_MAX) {
1186         Node* branchNode = m_block->at(branchIndexInBlock);
1187
1188         ASSERT(node->adjustedRefCount() == 1);
1189
1190         nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1191
1192         m_indexInBlock = branchIndexInBlock;
1193         m_currentNode = branchNode;
1194
1195         return true;
1196     }
1197
1198     nonSpeculativeNonPeepholeStrictEq(node, invert);
1199
1200     return false;
1201 }
1202
1203 static const char* dataFormatString(DataFormat format)
1204 {
1205     // These values correspond to the DataFormat enum.
1206     const char* strings[] = {
1207         "[  ]",
1208         "[ i]",
1209         "[ d]",
1210         "[ c]",
1211         "Err!",
1212         "Err!",
1213         "Err!",
1214         "Err!",
1215         "[J ]",
1216         "[Ji]",
1217         "[Jd]",
1218         "[Jc]",
1219         "Err!",
1220         "Err!",
1221         "Err!",
1222         "Err!",
1223     };
1224     return strings[format];
1225 }
1226
1227 void SpeculativeJIT::dump(const char* label)
1228 {
1229     if (label)
1230         dataLogF("<%s>\n", label);
1231
1232     dataLogF("  gprs:\n");
1233     m_gprs.dump();
1234     dataLogF("  fprs:\n");
1235     m_fprs.dump();
1236     dataLogF("  VirtualRegisters:\n");
1237     for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1238         GenerationInfo& info = m_generationInfo[i];
1239         if (info.alive())
1240             dataLogF("    % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1241         else
1242             dataLogF("    % 3d:[__][__]", i);
1243         if (info.registerFormat() == DataFormatDouble)
1244             dataLogF(":fpr%d\n", info.fpr());
1245         else if (info.registerFormat() != DataFormatNone
1246 #if USE(JSVALUE32_64)
1247             && !(info.registerFormat() & DataFormatJS)
1248 #endif
1249             ) {
1250             ASSERT(info.gpr() != InvalidGPRReg);
1251             dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1252         } else
1253             dataLogF("\n");
1254     }
1255     if (label)
1256         dataLogF("</%s>\n", label);
1257 }
1258
1259 GPRTemporary::GPRTemporary()
1260     : m_jit(0)
1261     , m_gpr(InvalidGPRReg)
1262 {
1263 }
1264
1265 GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1266     : m_jit(jit)
1267     , m_gpr(InvalidGPRReg)
1268 {
1269     m_gpr = m_jit->allocate();
1270 }
1271
1272 GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1273     : m_jit(jit)
1274     , m_gpr(InvalidGPRReg)
1275 {
1276     m_gpr = m_jit->allocate(specific);
1277 }
1278
1279 #if USE(JSVALUE32_64)
1280 GPRTemporary::GPRTemporary(
1281     SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1282     : m_jit(jit)
1283     , m_gpr(InvalidGPRReg)
1284 {
1285     if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1286         m_gpr = m_jit->reuse(op1.gpr(which));
1287     else
1288         m_gpr = m_jit->allocate();
1289 }
1290 #endif // USE(JSVALUE32_64)
1291
1292 JSValueRegsTemporary::JSValueRegsTemporary() { }
1293
1294 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1295 #if USE(JSVALUE64)
1296     : m_gpr(jit)
1297 #else
1298     : m_payloadGPR(jit)
1299     , m_tagGPR(jit)
1300 #endif
1301 {
1302 }
1303
1304 #if USE(JSVALUE64)
1305 template<typename T>
1306 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1307     : m_gpr(jit, Reuse, operand)
1308 {
1309 }
1310 #else
1311 template<typename T>
1312 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1313 {
1314     if (resultWord == PayloadWord) {
1315         m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1316         m_tagGPR = GPRTemporary(jit);
1317     } else {
1318         m_payloadGPR = GPRTemporary(jit);
1319         m_tagGPR = GPRTemporary(jit, Reuse, operand);
1320     }
1321 }
1322 #endif
1323
1324 #if USE(JSVALUE64)
1325 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1326 {
1327     m_gpr = GPRTemporary(jit, Reuse, operand);
1328 }
1329 #else
1330 JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1331 {
1332     if (jit->canReuse(operand.node())) {
1333         m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1334         m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1335     } else {
1336         m_payloadGPR = GPRTemporary(jit);
1337         m_tagGPR = GPRTemporary(jit);
1338     }
1339 }
1340 #endif
1341
1342 JSValueRegsTemporary::~JSValueRegsTemporary() { }
1343
1344 JSValueRegs JSValueRegsTemporary::regs()
1345 {
1346 #if USE(JSVALUE64)
1347     return JSValueRegs(m_gpr.gpr());
1348 #else
1349     return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1350 #endif
1351 }
1352
1353 void GPRTemporary::adopt(GPRTemporary& other)
1354 {
1355     ASSERT(!m_jit);
1356     ASSERT(m_gpr == InvalidGPRReg);
1357     ASSERT(other.m_jit);
1358     ASSERT(other.m_gpr != InvalidGPRReg);
1359     m_jit = other.m_jit;
1360     m_gpr = other.m_gpr;
1361     other.m_jit = 0;
1362     other.m_gpr = InvalidGPRReg;
1363 }
1364
1365 FPRTemporary::FPRTemporary(FPRTemporary&& other)
1366 {
1367     ASSERT(other.m_jit);
1368     ASSERT(other.m_fpr != InvalidFPRReg);
1369     m_jit = other.m_jit;
1370     m_fpr = other.m_fpr;
1371
1372     other.m_jit = nullptr;
1373 }
1374
1375 FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1376     : m_jit(jit)
1377     , m_fpr(InvalidFPRReg)
1378 {
1379     m_fpr = m_jit->fprAllocate();
1380 }
1381
1382 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1383     : m_jit(jit)
1384     , m_fpr(InvalidFPRReg)
1385 {
1386     if (m_jit->canReuse(op1.node()))
1387         m_fpr = m_jit->reuse(op1.fpr());
1388     else
1389         m_fpr = m_jit->fprAllocate();
1390 }
1391
1392 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1393     : m_jit(jit)
1394     , m_fpr(InvalidFPRReg)
1395 {
1396     if (m_jit->canReuse(op1.node()))
1397         m_fpr = m_jit->reuse(op1.fpr());
1398     else if (m_jit->canReuse(op2.node()))
1399         m_fpr = m_jit->reuse(op2.fpr());
1400     else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1401         m_fpr = m_jit->reuse(op1.fpr());
1402     else
1403         m_fpr = m_jit->fprAllocate();
1404 }
1405
1406 #if USE(JSVALUE32_64)
1407 FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1408     : m_jit(jit)
1409     , m_fpr(InvalidFPRReg)
1410 {
1411     if (op1.isDouble() && m_jit->canReuse(op1.node()))
1412         m_fpr = m_jit->reuse(op1.fpr());
1413     else
1414         m_fpr = m_jit->fprAllocate();
1415 }
1416 #endif
1417
1418 void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1419 {
1420     BasicBlock* taken = branchNode->branchData()->taken.block;
1421     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1422
1423     if (taken == nextBlock()) {
1424         condition = MacroAssembler::invert(condition);
1425         std::swap(taken, notTaken);
1426     }
1427
1428     SpeculateDoubleOperand op1(this, node->child1());
1429     SpeculateDoubleOperand op2(this, node->child2());
1430
1431     branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1432     jump(notTaken);
1433 }
1434
1435 void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1436 {
1437     BasicBlock* taken = branchNode->branchData()->taken.block;
1438     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1439
1440     MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1441
1442     if (taken == nextBlock()) {
1443         condition = MacroAssembler::NotEqual;
1444         BasicBlock* tmp = taken;
1445         taken = notTaken;
1446         notTaken = tmp;
1447     }
1448
1449     SpeculateCellOperand op1(this, node->child1());
1450     SpeculateCellOperand op2(this, node->child2());
1451
1452     GPRReg op1GPR = op1.gpr();
1453     GPRReg op2GPR = op2.gpr();
1454
1455     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1456         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1457             speculationCheck(
1458                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1459         }
1460         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1461             speculationCheck(
1462                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1463         }
1464     } else {
1465         if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1466             speculationCheck(
1467                 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1468                 m_jit.branchIfNotObject(op1GPR));
1469         }
1470         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1471             m_jit.branchTest8(
1472                 MacroAssembler::NonZero,
1473                 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
1474                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1475
1476         if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1477             speculationCheck(
1478                 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1479                 m_jit.branchIfNotObject(op2GPR));
1480         }
1481         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1482             m_jit.branchTest8(
1483                 MacroAssembler::NonZero,
1484                 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
1485                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1486     }
1487
1488     branchPtr(condition, op1GPR, op2GPR, taken);
1489     jump(notTaken);
1490 }
1491
1492 void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1493 {
1494     BasicBlock* taken = branchNode->branchData()->taken.block;
1495     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1496
1497     // The branch instruction will branch to the taken block.
1498     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1499     if (taken == nextBlock()) {
1500         condition = JITCompiler::invert(condition);
1501         BasicBlock* tmp = taken;
1502         taken = notTaken;
1503         notTaken = tmp;
1504     }
1505
1506     if (node->child1()->isInt32Constant()) {
1507         int32_t imm = node->child1()->asInt32();
1508         SpeculateBooleanOperand op2(this, node->child2());
1509         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1510     } else if (node->child2()->isInt32Constant()) {
1511         SpeculateBooleanOperand op1(this, node->child1());
1512         int32_t imm = node->child2()->asInt32();
1513         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1514     } else {
1515         SpeculateBooleanOperand op1(this, node->child1());
1516         SpeculateBooleanOperand op2(this, node->child2());
1517         branch32(condition, op1.gpr(), op2.gpr(), taken);
1518     }
1519
1520     jump(notTaken);
1521 }
1522
1523 void SpeculativeJIT::compileStringSlice(Node* node)
1524 {
1525     SpeculateCellOperand string(this, node->child1());
1526     GPRTemporary startIndex(this);
1527     GPRTemporary temp(this);
1528     GPRTemporary temp2(this);
1529
1530     GPRReg stringGPR = string.gpr();
1531     GPRReg startIndexGPR = startIndex.gpr();
1532     GPRReg tempGPR = temp.gpr();
1533     GPRReg temp2GPR = temp2.gpr();
1534
1535     speculateString(node->child1(), stringGPR);
1536
1537     {
1538         m_jit.load32(JITCompiler::Address(stringGPR, JSString::offsetOfLength()), temp2GPR);
1539
1540         emitPopulateSliceIndex(node->child2(), temp2GPR, startIndexGPR);
1541         if (node->child3())
1542             emitPopulateSliceIndex(node->child3(), temp2GPR, tempGPR);
1543         else
1544             m_jit.move(temp2GPR, tempGPR);
1545     }
1546
1547     CCallHelpers::JumpList doneCases;
1548     CCallHelpers::JumpList slowCases;
1549
1550     auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1551     m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1552     doneCases.append(m_jit.jump());
1553
1554     nonEmptyCase.link(&m_jit);
1555     m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1556     slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1557
1558     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1559     slowCases.append(m_jit.branchTestPtr(MacroAssembler::Zero, temp2GPR));
1560
1561     m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1562
1563     // Load the character into scratchReg
1564     m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1565     auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1566
1567     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1568     auto cont8Bit = m_jit.jump();
1569
1570     is16Bit.link(&m_jit);
1571     m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1572
1573     auto bigCharacter = m_jit.branch32(MacroAssembler::AboveOrEqual, tempGPR, TrustedImm32(0x100));
1574
1575     // 8 bit string values don't need the isASCII check.
1576     cont8Bit.link(&m_jit);
1577
1578     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1579     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1580     m_jit.loadPtr(tempGPR, tempGPR);
1581
1582     addSlowPathGenerator(
1583         slowPathCall(
1584             bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1585
1586     addSlowPathGenerator(
1587         slowPathCall(
1588             slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1589
1590     doneCases.link(&m_jit);
1591     cellResult(tempGPR, node);
1592 }
1593
1594 void SpeculativeJIT::compileToLowerCase(Node* node)
1595 {
1596     ASSERT(node->op() == ToLowerCase);
1597     SpeculateCellOperand string(this, node->child1());
1598     GPRTemporary temp(this);
1599     GPRTemporary index(this);
1600     GPRTemporary charReg(this);
1601     GPRTemporary length(this);
1602
1603     GPRReg stringGPR = string.gpr();
1604     GPRReg tempGPR = temp.gpr();
1605     GPRReg indexGPR = index.gpr();
1606     GPRReg charGPR = charReg.gpr();
1607     GPRReg lengthGPR = length.gpr();
1608
1609     speculateString(node->child1(), stringGPR);
1610
1611     CCallHelpers::JumpList slowPath;
1612
1613     m_jit.move(TrustedImmPtr(0), indexGPR);
1614
1615     m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1616     slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR));
1617
1618     slowPath.append(m_jit.branchTest32(
1619         MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1620         MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1621     m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1622     m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1623
1624     auto loopStart = m_jit.label();
1625     auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1626     m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1627     slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1628     m_jit.sub32(TrustedImm32('A'), charGPR);
1629     slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1630
1631     m_jit.add32(TrustedImm32(1), indexGPR);
1632     m_jit.jump().linkTo(loopStart, &m_jit);
1633
1634     slowPath.link(&m_jit);
1635     silentSpillAllRegisters(lengthGPR);
1636     callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1637     silentFillAllRegisters();
1638     m_jit.exceptionCheck();
1639     auto done = m_jit.jump();
1640
1641     loopDone.link(&m_jit);
1642     m_jit.move(stringGPR, lengthGPR);
1643
1644     done.link(&m_jit);
1645     cellResult(lengthGPR, node);
1646 }
1647
1648 void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1649 {
1650     BasicBlock* taken = branchNode->branchData()->taken.block;
1651     BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1652
1653     // The branch instruction will branch to the taken block.
1654     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1655     if (taken == nextBlock()) {
1656         condition = JITCompiler::invert(condition);
1657         BasicBlock* tmp = taken;
1658         taken = notTaken;
1659         notTaken = tmp;
1660     }
1661
1662     if (node->child1()->isInt32Constant()) {
1663         int32_t imm = node->child1()->asInt32();
1664         SpeculateInt32Operand op2(this, node->child2());
1665         branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1666     } else if (node->child2()->isInt32Constant()) {
1667         SpeculateInt32Operand op1(this, node->child1());
1668         int32_t imm = node->child2()->asInt32();
1669         branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1670     } else {
1671         SpeculateInt32Operand op1(this, node->child1());
1672         SpeculateInt32Operand op2(this, node->child2());
1673         branch32(condition, op1.gpr(), op2.gpr(), taken);
1674     }
1675
1676     jump(notTaken);
1677 }
1678
1679 // Returns true if the compare is fused with a subsequent branch.
1680 bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1681 {
1682     // Fused compare & branch.
1683     unsigned branchIndexInBlock = detectPeepHoleBranch();
1684     if (branchIndexInBlock != UINT_MAX) {
1685         Node* branchNode = m_block->at(branchIndexInBlock);
1686
1687         // detectPeepHoleBranch currently only permits the branch to be the very next node,
1688         // so can be no intervening nodes to also reference the compare.
1689         ASSERT(node->adjustedRefCount() == 1);
1690
1691         if (node->isBinaryUseKind(Int32Use))
1692             compilePeepHoleInt32Branch(node, branchNode, condition);
1693 #if USE(JSVALUE64)
1694         else if (node->isBinaryUseKind(Int52RepUse))
1695             compilePeepHoleInt52Branch(node, branchNode, condition);
1696 #endif // USE(JSVALUE64)
1697         else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1698             // Use non-peephole comparison, for now.
1699             return false;
1700         } else if (node->isBinaryUseKind(DoubleRepUse))
1701             compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1702         else if (node->op() == CompareEq) {
1703             if (node->isBinaryUseKind(BooleanUse))
1704                 compilePeepHoleBooleanBranch(node, branchNode, condition);
1705             else if (node->isBinaryUseKind(SymbolUse))
1706                 compilePeepHoleSymbolEquality(node, branchNode);
1707             else if (node->isBinaryUseKind(ObjectUse))
1708                 compilePeepHoleObjectEquality(node, branchNode);
1709             else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1710                 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1711             else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1712                 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1713             else if (!needsTypeCheck(node->child1(), SpecOther))
1714                 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1715             else if (!needsTypeCheck(node->child2(), SpecOther))
1716                 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1717             else {
1718                 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1719                 return true;
1720             }
1721         } else {
1722             nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1723             return true;
1724         }
1725
1726         use(node->child1());
1727         use(node->child2());
1728         m_indexInBlock = branchIndexInBlock;
1729         m_currentNode = branchNode;
1730         return true;
1731     }
1732     return false;
1733 }
1734
1735 void SpeculativeJIT::noticeOSRBirth(Node* node)
1736 {
1737     if (!node->hasVirtualRegister())
1738         return;
1739
1740     VirtualRegister virtualRegister = node->virtualRegister();
1741     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1742
1743     info.noticeOSRBirth(*m_stream, node, virtualRegister);
1744 }
1745
1746 void SpeculativeJIT::compileMovHint(Node* node)
1747 {
1748     ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1749
1750     Node* child = node->child1().node();
1751     noticeOSRBirth(child);
1752
1753     m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1754 }
1755
1756 void SpeculativeJIT::bail(AbortReason reason)
1757 {
1758     if (verboseCompilationEnabled())
1759         dataLog("Bailing compilation.\n");
1760     m_compileOkay = true;
1761     m_jit.abortWithReason(reason, m_lastGeneratedNode);
1762     clearGenerationInfo();
1763 }
1764
1765 void SpeculativeJIT::compileCurrentBlock()
1766 {
1767     ASSERT(m_compileOkay);
1768
1769     if (!m_block)
1770         return;
1771
1772     ASSERT(m_block->isReachable);
1773
1774     m_jit.blockHeads()[m_block->index] = m_jit.label();
1775
1776     if (!m_block->intersectionOfCFAHasVisited) {
1777         // Don't generate code for basic blocks that are unreachable according to CFA.
1778         // But to be sure that nobody has generated a jump to this block, drop in a
1779         // breakpoint here.
1780         m_jit.abortWithReason(DFGUnreachableBasicBlock);
1781         return;
1782     }
1783
1784     if (m_block->isCatchEntrypoint) {
1785         m_jit.addPtr(CCallHelpers::TrustedImm32(m_jit.graph().stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister,  CCallHelpers::stackPointerRegister);
1786         m_jit.emitSaveCalleeSaves();
1787         m_jit.emitMaterializeTagCheckRegisters();
1788         m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1789     }
1790
1791     m_stream->appendAndLog(VariableEvent::reset());
1792
1793     m_jit.jitAssertHasValidCallFrame();
1794     m_jit.jitAssertTagsInPlace();
1795     m_jit.jitAssertArgumentCountSane();
1796
1797     m_state.reset();
1798     m_state.beginBasicBlock(m_block);
1799
1800     for (size_t i = m_block->variablesAtHead.size(); i--;) {
1801         int operand = m_block->variablesAtHead.operandForIndex(i);
1802         Node* node = m_block->variablesAtHead[i];
1803         if (!node)
1804             continue; // No need to record dead SetLocal's.
1805
1806         VariableAccessData* variable = node->variableAccessData();
1807         DataFormat format;
1808         if (!node->refCount())
1809             continue; // No need to record dead SetLocal's.
1810         format = dataFormatFor(variable->flushFormat());
1811         m_stream->appendAndLog(
1812             VariableEvent::setLocal(
1813                 VirtualRegister(operand),
1814                 variable->machineLocal(),
1815                 format));
1816     }
1817
1818     m_origin = NodeOrigin();
1819
1820     for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1821         m_currentNode = m_block->at(m_indexInBlock);
1822
1823         // We may have hit a contradiction that the CFA was aware of but that the JIT
1824         // didn't cause directly.
1825         if (!m_state.isValid()) {
1826             bail(DFGBailedAtTopOfBlock);
1827             return;
1828         }
1829
1830         m_interpreter.startExecuting();
1831         m_interpreter.executeKnownEdgeTypes(m_currentNode);
1832         m_jit.setForNode(m_currentNode);
1833         m_origin = m_currentNode->origin;
1834         if (validationEnabled())
1835             m_origin.exitOK &= mayExit(m_jit.graph(), m_currentNode) == Exits;
1836         m_lastGeneratedNode = m_currentNode->op();
1837
1838         ASSERT(m_currentNode->shouldGenerate());
1839
1840         if (verboseCompilationEnabled()) {
1841             dataLogF(
1842                 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1843                 (int)m_currentNode->index(),
1844                 m_currentNode->origin.semantic.bytecodeIndex, m_jit.debugOffset());
1845             dataLog("\n");
1846         }
1847
1848         if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1849             m_jit.jitReleaseAssertNoException(*m_jit.vm());
1850
1851         m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1852
1853         compile(m_currentNode);
1854
1855         if (belongsInMinifiedGraph(m_currentNode->op()))
1856             m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1857
1858 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1859         m_jit.clearRegisterAllocationOffsets();
1860 #endif
1861
1862         if (!m_compileOkay) {
1863             bail(DFGBailedAtEndOfNode);
1864             return;
1865         }
1866
1867         // Make sure that the abstract state is rematerialized for the next node.
1868         m_interpreter.executeEffects(m_indexInBlock);
1869     }
1870
1871     // Perform the most basic verification that children have been used correctly.
1872     if (!ASSERT_DISABLED) {
1873         for (auto& info : m_generationInfo)
1874             RELEASE_ASSERT(!info.alive());
1875     }
1876 }
1877
1878 // If we are making type predictions about our arguments then
1879 // we need to check that they are correct on function entry.
1880 void SpeculativeJIT::checkArgumentTypes()
1881 {
1882     ASSERT(!m_currentNode);
1883     m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1884
1885     auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1886     for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1887         Node* node = arguments[i];
1888         if (!node) {
1889             // The argument is dead. We don't do any checks for such arguments.
1890             continue;
1891         }
1892
1893         ASSERT(node->op() == SetArgument);
1894         ASSERT(node->shouldGenerate());
1895
1896         VariableAccessData* variableAccessData = node->variableAccessData();
1897         FlushFormat format = variableAccessData->flushFormat();
1898
1899         if (format == FlushedJSValue)
1900             continue;
1901
1902         VirtualRegister virtualRegister = variableAccessData->local();
1903
1904         JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1905
1906 #if USE(JSVALUE64)
1907         switch (format) {
1908         case FlushedInt32: {
1909             speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1910             break;
1911         }
1912         case FlushedBoolean: {
1913             GPRTemporary temp(this);
1914             m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1915             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1916             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1917             break;
1918         }
1919         case FlushedCell: {
1920             speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1921             break;
1922         }
1923         default:
1924             RELEASE_ASSERT_NOT_REACHED();
1925             break;
1926         }
1927 #else
1928         switch (format) {
1929         case FlushedInt32: {
1930             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1931             break;
1932         }
1933         case FlushedBoolean: {
1934             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1935             break;
1936         }
1937         case FlushedCell: {
1938             speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1939             break;
1940         }
1941         default:
1942             RELEASE_ASSERT_NOT_REACHED();
1943             break;
1944         }
1945 #endif
1946     }
1947
1948     m_origin = NodeOrigin();
1949 }
1950
1951 bool SpeculativeJIT::compile()
1952 {
1953     checkArgumentTypes();
1954
1955     ASSERT(!m_currentNode);
1956     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1957         m_jit.setForBlockIndex(blockIndex);
1958         m_block = m_jit.graph().block(blockIndex);
1959         compileCurrentBlock();
1960     }
1961     linkBranches();
1962     return true;
1963 }
1964
1965 void SpeculativeJIT::createOSREntries()
1966 {
1967     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1968         BasicBlock* block = m_jit.graph().block(blockIndex);
1969         if (!block)
1970             continue;
1971         if (block->isOSRTarget || block->isCatchEntrypoint) {
1972             // Currently we don't have OSR entry trampolines. We could add them
1973             // here if need be.
1974             m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1975         }
1976     }
1977 }
1978
1979 void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
1980 {
1981     unsigned osrEntryIndex = 0;
1982     for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1983         BasicBlock* block = m_jit.graph().block(blockIndex);
1984         if (!block)
1985             continue;
1986         if (!block->isOSRTarget && !block->isCatchEntrypoint)
1987             continue;
1988         if (block->isCatchEntrypoint) {
1989             auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
1990             Vector<FlushFormat> argumentFormats;
1991             argumentFormats.reserveInitialCapacity(argumentsVector.size());
1992             for (Node* setArgument : argumentsVector) {
1993                 if (setArgument) {
1994                     FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
1995                     ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
1996                     argumentFormats.uncheckedAppend(flushFormat);
1997                 } else
1998                     argumentFormats.uncheckedAppend(DeadFlush);
1999             }
2000             m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
2001         } else {
2002             ASSERT(block->isOSRTarget);
2003             m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
2004         }
2005     }
2006
2007     m_jit.jitCode()->finalizeOSREntrypoints();
2008     m_jit.jitCode()->common.finalizeCatchEntrypoints();
2009
2010     ASSERT(osrEntryIndex == m_osrEntryHeads.size());
2011
2012     if (verboseCompilationEnabled()) {
2013         DumpContext dumpContext;
2014         dataLog("OSR Entries:\n");
2015         for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
2016             dataLog("    ", inContext(entryData, &dumpContext), "\n");
2017         if (!dumpContext.isEmpty())
2018             dumpContext.dump(WTF::dataFile());
2019     }
2020 }
2021
2022 void SpeculativeJIT::compileCheckTraps(Node*)
2023 {
2024     ASSERT(Options::usePollingTraps());
2025     GPRTemporary unused(this);
2026     GPRReg unusedGPR = unused.gpr();
2027
2028     JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2029         JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2030
2031     addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2032 }
2033
2034 void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2035 {
2036     Edge child3 = m_jit.graph().varArgChild(node, 2);
2037     Edge child4 = m_jit.graph().varArgChild(node, 3);
2038
2039     ArrayMode arrayMode = node->arrayMode();
2040
2041     GPRReg baseReg = base.gpr();
2042     GPRReg propertyReg = property.gpr();
2043
2044     SpeculateDoubleOperand value(this, child3);
2045
2046     FPRReg valueReg = value.fpr();
2047
2048     DFG_TYPE_CHECK(
2049         JSValueRegs(), child3, SpecFullRealNumber,
2050         m_jit.branchDouble(
2051             MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg));
2052
2053     if (!m_compileOkay)
2054         return;
2055
2056     StorageOperand storage(this, child4);
2057     GPRReg storageReg = storage.gpr();
2058
2059     if (node->op() == PutByValAlias) {
2060         // Store the value to the array.
2061         GPRReg propertyReg = property.gpr();
2062         FPRReg valueReg = value.fpr();
2063         m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2064
2065         noResult(m_currentNode);
2066         return;
2067     }
2068
2069     GPRTemporary temporary;
2070     GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2071
2072     MacroAssembler::Jump slowCase;
2073
2074     if (arrayMode.isInBounds()) {
2075         speculationCheck(
2076             OutOfBounds, JSValueRegs(), 0,
2077             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2078     } else {
2079         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2080
2081         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2082
2083         if (!arrayMode.isOutOfBounds())
2084             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2085
2086         m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2087         m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2088
2089         inBounds.link(&m_jit);
2090     }
2091
2092     m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2093
2094     base.use();
2095     property.use();
2096     value.use();
2097     storage.use();
2098
2099     if (arrayMode.isOutOfBounds()) {
2100         addSlowPathGenerator(
2101             slowPathCall(
2102                 slowCase, this,
2103                 m_jit.codeBlock()->isStrictMode() ? operationPutDoubleByValBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2104                 NoResult, baseReg, propertyReg, valueReg));
2105     }
2106
2107     noResult(m_currentNode, UseChildrenCalledExplicitly);
2108 }
2109
2110 void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2111 {
2112     SpeculateCellOperand string(this, node->child1());
2113     SpeculateStrictInt32Operand index(this, node->child2());
2114     StorageOperand storage(this, node->child3());
2115
2116     GPRReg stringReg = string.gpr();
2117     GPRReg indexReg = index.gpr();
2118     GPRReg storageReg = storage.gpr();
2119
2120     ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2121
2122     // unsigned comparison so we can filter out negative indices and indices that are too large
2123     speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength())));
2124
2125     GPRTemporary scratch(this);
2126     GPRReg scratchReg = scratch.gpr();
2127
2128     m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2129     m_jit.and32(MacroAssembler::Address(scratchReg, StringImpl::maskOffset()), indexReg);
2130
2131     // Load the character into scratchReg
2132     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2133
2134     m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2135     JITCompiler::Jump cont8Bit = m_jit.jump();
2136
2137     is16Bit.link(&m_jit);
2138
2139     m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2140
2141     cont8Bit.link(&m_jit);
2142
2143     int32Result(scratchReg, m_currentNode);
2144 }
2145
2146 void SpeculativeJIT::compileGetByValOnString(Node* node)
2147 {
2148     SpeculateCellOperand base(this, node->child1());
2149     SpeculateStrictInt32Operand property(this, node->child2());
2150     StorageOperand storage(this, node->child3());
2151     GPRReg baseReg = base.gpr();
2152     GPRReg propertyReg = property.gpr();
2153     GPRReg storageReg = storage.gpr();
2154
2155     GPRTemporary scratch(this);
2156     GPRReg scratchReg = scratch.gpr();
2157 #if USE(JSVALUE32_64)
2158     GPRTemporary resultTag;
2159     GPRReg resultTagReg = InvalidGPRReg;
2160     if (node->arrayMode().isOutOfBounds()) {
2161         GPRTemporary realResultTag(this);
2162         resultTag.adopt(realResultTag);
2163         resultTagReg = resultTag.gpr();
2164     }
2165 #endif
2166
2167     ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2168
2169     // unsigned comparison so we can filter out negative indices and indices that are too large
2170     JITCompiler::Jump outOfBounds = m_jit.branch32(
2171         MacroAssembler::AboveOrEqual, propertyReg,
2172         MacroAssembler::Address(baseReg, JSString::offsetOfLength()));
2173     if (node->arrayMode().isInBounds())
2174         speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2175
2176     m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2177     m_jit.and32(MacroAssembler::Address(scratchReg, StringImpl::maskOffset()), propertyReg);
2178
2179     // Load the character into scratchReg
2180     JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2181
2182     m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2183     JITCompiler::Jump cont8Bit = m_jit.jump();
2184
2185     is16Bit.link(&m_jit);
2186
2187     m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2188
2189     JITCompiler::Jump bigCharacter =
2190         m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100));
2191
2192     // 8 bit string values don't need the isASCII check.
2193     cont8Bit.link(&m_jit);
2194
2195     m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2196     m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2197     m_jit.loadPtr(scratchReg, scratchReg);
2198
2199     addSlowPathGenerator(
2200         slowPathCall(
2201             bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2202
2203     if (node->arrayMode().isOutOfBounds()) {
2204 #if USE(JSVALUE32_64)
2205         m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2206 #endif
2207
2208         JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2209         bool prototypeChainIsSane = false;
2210         if (globalObject->stringPrototypeChainIsSane()) {
2211             // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2212             // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2213             // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2214             // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2215             // indexed properties either.
2216             // https://bugs.webkit.org/show_bug.cgi?id=144668
2217             m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure());
2218             m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure());
2219             prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2220         }
2221         if (prototypeChainIsSane) {
2222 #if USE(JSVALUE64)
2223             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2224                 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2225 #else
2226             addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2227                 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2228                 baseReg, propertyReg));
2229 #endif
2230         } else {
2231 #if USE(JSVALUE64)
2232             addSlowPathGenerator(
2233                 slowPathCall(
2234                     outOfBounds, this, operationGetByValStringInt,
2235                     scratchReg, baseReg, propertyReg));
2236 #else
2237             addSlowPathGenerator(
2238                 slowPathCall(
2239                     outOfBounds, this, operationGetByValStringInt,
2240                     JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2241 #endif
2242         }
2243
2244 #if USE(JSVALUE64)
2245         jsValueResult(scratchReg, m_currentNode);
2246 #else
2247         jsValueResult(resultTagReg, scratchReg, m_currentNode);
2248 #endif
2249     } else
2250         cellResult(scratchReg, m_currentNode);
2251 }
2252
2253 void SpeculativeJIT::compileFromCharCode(Node* node)
2254 {
2255     Edge& child = node->child1();
2256     if (child.useKind() == UntypedUse) {
2257         JSValueOperand opr(this, child);
2258         JSValueRegs oprRegs = opr.jsValueRegs();
2259
2260         flushRegisters();
2261         JSValueRegsFlushedCallResult result(this);
2262         JSValueRegs resultRegs = result.regs();
2263         callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2264         m_jit.exceptionCheck();
2265
2266         jsValueResult(resultRegs, node);
2267         return;
2268     }
2269
2270     SpeculateStrictInt32Operand property(this, child);
2271     GPRReg propertyReg = property.gpr();
2272     GPRTemporary smallStrings(this);
2273     GPRTemporary scratch(this);
2274     GPRReg scratchReg = scratch.gpr();
2275     GPRReg smallStringsReg = smallStrings.gpr();
2276
2277     JITCompiler::JumpList slowCases;
2278     slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff)));
2279     m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2280     m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2281
2282     slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2283     addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2284     cellResult(scratchReg, m_currentNode);
2285 }
2286
2287 GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2288 {
2289     VirtualRegister virtualRegister = node->virtualRegister();
2290     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2291
2292     switch (info.registerFormat()) {
2293     case DataFormatStorage:
2294         RELEASE_ASSERT_NOT_REACHED();
2295
2296     case DataFormatBoolean:
2297     case DataFormatCell:
2298         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2299         return GeneratedOperandTypeUnknown;
2300
2301     case DataFormatNone:
2302     case DataFormatJSCell:
2303     case DataFormatJS:
2304     case DataFormatJSBoolean:
2305     case DataFormatJSDouble:
2306         return GeneratedOperandJSValue;
2307
2308     case DataFormatJSInt32:
2309     case DataFormatInt32:
2310         return GeneratedOperandInteger;
2311
2312     default:
2313         RELEASE_ASSERT_NOT_REACHED();
2314         return GeneratedOperandTypeUnknown;
2315     }
2316 }
2317
2318 void SpeculativeJIT::compileValueToInt32(Node* node)
2319 {
2320     switch (node->child1().useKind()) {
2321 #if USE(JSVALUE64)
2322     case Int52RepUse: {
2323         SpeculateStrictInt52Operand op1(this, node->child1());
2324         GPRTemporary result(this, Reuse, op1);
2325         GPRReg op1GPR = op1.gpr();
2326         GPRReg resultGPR = result.gpr();
2327         m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2328         int32Result(resultGPR, node, DataFormatInt32);
2329         return;
2330     }
2331 #endif // USE(JSVALUE64)
2332
2333     case DoubleRepUse: {
2334         GPRTemporary result(this);
2335         SpeculateDoubleOperand op1(this, node->child1());
2336         FPRReg fpr = op1.fpr();
2337         GPRReg gpr = result.gpr();
2338         JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2339
2340         addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2341             hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2342
2343         int32Result(gpr, node);
2344         return;
2345     }
2346
2347     case NumberUse:
2348     case NotCellUse: {
2349         switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2350         case GeneratedOperandInteger: {
2351             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2352             GPRTemporary result(this, Reuse, op1);
2353             m_jit.move(op1.gpr(), result.gpr());
2354             int32Result(result.gpr(), node, op1.format());
2355             return;
2356         }
2357         case GeneratedOperandJSValue: {
2358             GPRTemporary result(this);
2359 #if USE(JSVALUE64)
2360             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2361
2362             GPRReg gpr = op1.gpr();
2363             GPRReg resultGpr = result.gpr();
2364             FPRTemporary tempFpr(this);
2365             FPRReg fpr = tempFpr.fpr();
2366
2367             JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
2368             JITCompiler::JumpList converted;
2369
2370             if (node->child1().useKind() == NumberUse) {
2371                 DFG_TYPE_CHECK(
2372                     JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2373                     m_jit.branchTest64(
2374                         MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister));
2375             } else {
2376                 JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
2377
2378                 DFG_TYPE_CHECK(
2379                     JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2380
2381                 // It's not a cell: so true turns into 1 and all else turns into 0.
2382                 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2383                 converted.append(m_jit.jump());
2384
2385                 isNumber.link(&m_jit);
2386             }
2387
2388             // First, if we get here we have a double encoded as a JSValue
2389             unboxDouble(gpr, resultGpr, fpr);
2390
2391             silentSpillAllRegisters(resultGpr);
2392             callOperation(operationToInt32, resultGpr, fpr);
2393             silentFillAllRegisters();
2394
2395             converted.append(m_jit.jump());
2396
2397             isInteger.link(&m_jit);
2398             m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2399
2400             converted.link(&m_jit);
2401 #else
2402             Node* childNode = node->child1().node();
2403             VirtualRegister virtualRegister = childNode->virtualRegister();
2404             GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2405
2406             JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2407
2408             GPRReg payloadGPR = op1.payloadGPR();
2409             GPRReg resultGpr = result.gpr();
2410
2411             JITCompiler::JumpList converted;
2412
2413             if (info.registerFormat() == DataFormatJSInt32)
2414                 m_jit.move(payloadGPR, resultGpr);
2415             else {
2416                 GPRReg tagGPR = op1.tagGPR();
2417                 FPRTemporary tempFpr(this);
2418                 FPRReg fpr = tempFpr.fpr();
2419                 FPRTemporary scratch(this);
2420
2421                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
2422
2423                 if (node->child1().useKind() == NumberUse) {
2424                     DFG_TYPE_CHECK(
2425                         op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2426                         m_jit.branch32(
2427                             MacroAssembler::AboveOrEqual, tagGPR,
2428                             TrustedImm32(JSValue::LowestTag)));
2429                 } else {
2430                     JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2431
2432                     DFG_TYPE_CHECK(
2433                         op1.jsValueRegs(), node->child1(), ~SpecCell,
2434                         m_jit.branchIfCell(op1.jsValueRegs()));
2435
2436                     // It's not a cell: so true turns into 1 and all else turns into 0.
2437                     JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag));
2438                     m_jit.move(TrustedImm32(0), resultGpr);
2439                     converted.append(m_jit.jump());
2440
2441                     isBoolean.link(&m_jit);
2442                     m_jit.move(payloadGPR, resultGpr);
2443                     converted.append(m_jit.jump());
2444
2445                     isNumber.link(&m_jit);
2446                 }
2447
2448                 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2449
2450                 silentSpillAllRegisters(resultGpr);
2451                 callOperation(operationToInt32, resultGpr, fpr);
2452                 silentFillAllRegisters();
2453
2454                 converted.append(m_jit.jump());
2455
2456                 isInteger.link(&m_jit);
2457                 m_jit.move(payloadGPR, resultGpr);
2458
2459                 converted.link(&m_jit);
2460             }
2461 #endif
2462             int32Result(resultGpr, node);
2463             return;
2464         }
2465         case GeneratedOperandTypeUnknown:
2466             RELEASE_ASSERT(!m_compileOkay);
2467             return;
2468         }
2469         RELEASE_ASSERT_NOT_REACHED();
2470         return;
2471     }
2472
2473     default:
2474         ASSERT(!m_compileOkay);
2475         return;
2476     }
2477 }
2478
2479 void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2480 {
2481     if (doesOverflow(node->arithMode())) {
2482         if (enableInt52()) {
2483             SpeculateInt32Operand op1(this, node->child1());
2484             GPRTemporary result(this, Reuse, op1);
2485             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2486             strictInt52Result(result.gpr(), node);
2487             return;
2488         }
2489         SpeculateInt32Operand op1(this, node->child1());
2490         FPRTemporary result(this);
2491
2492         GPRReg inputGPR = op1.gpr();
2493         FPRReg outputFPR = result.fpr();
2494
2495         m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2496
2497         JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2498         m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2499         positive.link(&m_jit);
2500
2501         doubleResult(outputFPR, node);
2502         return;
2503     }
2504
2505     RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2506
2507     SpeculateInt32Operand op1(this, node->child1());
2508     GPRTemporary result(this);
2509
2510     m_jit.move(op1.gpr(), result.gpr());
2511
2512     speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2513
2514     int32Result(result.gpr(), node, op1.format());
2515 }
2516
2517 void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2518 {
2519     SpeculateDoubleOperand op1(this, node->child1());
2520     FPRTemporary scratch(this);
2521     GPRTemporary result(this);
2522
2523     FPRReg valueFPR = op1.fpr();
2524     FPRReg scratchFPR = scratch.fpr();
2525     GPRReg resultGPR = result.gpr();
2526
2527     JITCompiler::JumpList failureCases;
2528     RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2529     m_jit.branchConvertDoubleToInt32(
2530         valueFPR, resultGPR, failureCases, scratchFPR,
2531         shouldCheckNegativeZero(node->arithMode()));
2532     speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2533
2534     int32Result(resultGPR, node);
2535 }
2536
2537 void SpeculativeJIT::compileDoubleRep(Node* node)
2538 {
2539     switch (node->child1().useKind()) {
2540     case RealNumberUse: {
2541         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2542         FPRTemporary result(this);
2543
2544         JSValueRegs op1Regs = op1.jsValueRegs();
2545         FPRReg resultFPR = result.fpr();
2546
2547 #if USE(JSVALUE64)
2548         GPRTemporary temp(this);
2549         GPRReg tempGPR = temp.gpr();
2550         m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2551 #else
2552         FPRTemporary temp(this);
2553         FPRReg tempFPR = temp.fpr();
2554         unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2555 #endif
2556
2557         JITCompiler::Jump done = m_jit.branchDouble(
2558             JITCompiler::DoubleEqual, resultFPR, resultFPR);
2559
2560         DFG_TYPE_CHECK(
2561             op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2562         m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2563
2564         done.link(&m_jit);
2565
2566         doubleResult(resultFPR, node);
2567         return;
2568     }
2569
2570     case NotCellUse:
2571     case NumberUse: {
2572         ASSERT(!node->child1()->isNumberConstant()); // This should have been constant folded.
2573
2574         SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2575         if (isInt32Speculation(possibleTypes)) {
2576             SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2577             FPRTemporary result(this);
2578             m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2579             doubleResult(result.fpr(), node);
2580             return;
2581         }
2582
2583         JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2584         FPRTemporary result(this);
2585
2586 #if USE(JSVALUE64)
2587         GPRTemporary temp(this);
2588
2589         GPRReg op1GPR = op1.gpr();
2590         GPRReg tempGPR = temp.gpr();
2591         FPRReg resultFPR = result.fpr();
2592         JITCompiler::JumpList done;
2593
2594         JITCompiler::Jump isInteger = m_jit.branch64(
2595             MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister);
2596
2597         if (node->child1().useKind() == NotCellUse) {
2598             JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagTypeNumberRegister);
2599             JITCompiler::Jump isUndefined = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueUndefined));
2600
2601             static const double zero = 0;
2602             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2603
2604             JITCompiler::Jump isNull = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueNull));
2605             done.append(isNull);
2606
2607             DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2608                 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2609
2610             JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2611             static const double one = 1;
2612             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2613             done.append(m_jit.jump());
2614             done.append(isFalse);
2615
2616             isUndefined.link(&m_jit);
2617             static const double NaN = PNaN;
2618             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2619             done.append(m_jit.jump());
2620
2621             isNumber.link(&m_jit);
2622         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2623             typeCheck(
2624                 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2625                 m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister));
2626         }
2627
2628         unboxDouble(op1GPR, tempGPR, resultFPR);
2629         done.append(m_jit.jump());
2630
2631         isInteger.link(&m_jit);
2632         m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2633         done.link(&m_jit);
2634 #else // USE(JSVALUE64) -> this is the 32_64 case
2635         FPRTemporary temp(this);
2636
2637         GPRReg op1TagGPR = op1.tagGPR();
2638         GPRReg op1PayloadGPR = op1.payloadGPR();
2639         FPRReg tempFPR = temp.fpr();
2640         FPRReg resultFPR = result.fpr();
2641         JITCompiler::JumpList done;
2642
2643         JITCompiler::Jump isInteger = m_jit.branch32(
2644             MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag));
2645
2646         if (node->child1().useKind() == NotCellUse) {
2647             JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2648             JITCompiler::Jump isUndefined = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::UndefinedTag));
2649
2650             static const double zero = 0;
2651             m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2652
2653             JITCompiler::Jump isNull = m_jit.branch32(JITCompiler::Equal, op1TagGPR, TrustedImm32(JSValue::NullTag));
2654             done.append(isNull);
2655
2656             DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branch32(JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::BooleanTag)));
2657
2658             JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2659             static const double one = 1;
2660             m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2661             done.append(m_jit.jump());
2662             done.append(isFalse);
2663
2664             isUndefined.link(&m_jit);
2665             static const double NaN = PNaN;
2666             m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2667             done.append(m_jit.jump());
2668
2669             isNumber.link(&m_jit);
2670         } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2671             typeCheck(
2672                 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2673                 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2674         }
2675
2676         unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2677         done.append(m_jit.jump());
2678
2679         isInteger.link(&m_jit);
2680         m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2681         done.link(&m_jit);
2682 #endif // USE(JSVALUE64)
2683
2684         doubleResult(resultFPR, node);
2685         return;
2686     }
2687
2688 #if USE(JSVALUE64)
2689     case Int52RepUse: {
2690         SpeculateStrictInt52Operand value(this, node->child1());
2691         FPRTemporary result(this);
2692
2693         GPRReg valueGPR = value.gpr();
2694         FPRReg resultFPR = result.fpr();
2695
2696         m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2697
2698         doubleResult(resultFPR, node);
2699         return;
2700     }
2701 #endif // USE(JSVALUE64)
2702
2703     default:
2704         RELEASE_ASSERT_NOT_REACHED();
2705         return;
2706     }
2707 }
2708
2709 void SpeculativeJIT::compileValueRep(Node* node)
2710 {
2711     switch (node->child1().useKind()) {
2712     case DoubleRepUse: {
2713         SpeculateDoubleOperand value(this, node->child1());
2714         JSValueRegsTemporary result(this);
2715
2716         FPRReg valueFPR = value.fpr();
2717         JSValueRegs resultRegs = result.regs();
2718
2719         // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2720         // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2721         // subject to a prior SetLocal, filtering the value would imply that the corresponding
2722         // local was purified.
2723         if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2724             m_jit.purifyNaN(valueFPR);
2725
2726         boxDouble(valueFPR, resultRegs);
2727
2728         jsValueResult(resultRegs, node);
2729         return;
2730     }
2731
2732 #if USE(JSVALUE64)
2733     case Int52RepUse: {
2734         SpeculateStrictInt52Operand value(this, node->child1());
2735         GPRTemporary result(this);
2736
2737         GPRReg valueGPR = value.gpr();
2738         GPRReg resultGPR = result.gpr();
2739
2740         boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2741
2742         jsValueResult(resultGPR, node);
2743         return;
2744     }
2745 #endif // USE(JSVALUE64)
2746
2747     default:
2748         RELEASE_ASSERT_NOT_REACHED();
2749         return;
2750     }
2751 }
2752
2753 static double clampDoubleToByte(double d)
2754 {
2755     d += 0.5;
2756     if (!(d > 0))
2757         d = 0;
2758     else if (d > 255)
2759         d = 255;
2760     return d;
2761 }
2762
2763 static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2764 {
2765     MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2766     MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2767     jit.xorPtr(result, result);
2768     MacroAssembler::Jump clamped = jit.jump();
2769     tooBig.link(&jit);
2770     jit.move(JITCompiler::TrustedImm32(255), result);
2771     clamped.link(&jit);
2772     inBounds.link(&jit);
2773 }
2774
2775 static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2776 {
2777     // Unordered compare so we pick up NaN
2778     static const double zero = 0;
2779     static const double byteMax = 255;
2780     static const double half = 0.5;
2781     jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2782     MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2783     jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2784     MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2785
2786     jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2787     // FIXME: This should probably just use a floating point round!
2788     // https://bugs.webkit.org/show_bug.cgi?id=72054
2789     jit.addDouble(source, scratch);
2790     jit.truncateDoubleToInt32(scratch, result);
2791     MacroAssembler::Jump truncatedInt = jit.jump();
2792
2793     tooSmall.link(&jit);
2794     jit.xorPtr(result, result);
2795     MacroAssembler::Jump zeroed = jit.jump();
2796
2797     tooBig.link(&jit);
2798     jit.move(JITCompiler::TrustedImm32(255), result);
2799
2800     truncatedInt.link(&jit);
2801     zeroed.link(&jit);
2802
2803 }
2804
2805 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2806 {
2807     if (node->op() == PutByValAlias)
2808         return JITCompiler::Jump();
2809     JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2810         m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2811     if (view) {
2812         uint32_t length = view->length();
2813         Node* indexNode = m_jit.graph().child(node, 1).node();
2814         if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2815             return JITCompiler::Jump();
2816         return m_jit.branch32(
2817             MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2818     }
2819     return m_jit.branch32(
2820         MacroAssembler::AboveOrEqual, indexGPR,
2821         MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2822 }
2823
2824 void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2825 {
2826     JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2827     if (!jump.isSet())
2828         return;
2829     speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2830 }
2831
2832 JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2833 {
2834     JITCompiler::Jump done;
2835     if (outOfBounds.isSet()) {
2836         done = m_jit.jump();
2837         if (node->arrayMode().isInBounds())
2838             speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2839         else {
2840             outOfBounds.link(&m_jit);
2841
2842             JITCompiler::Jump notWasteful = m_jit.branch32(
2843                 MacroAssembler::NotEqual,
2844                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2845                 TrustedImm32(WastefulTypedArray));
2846
2847             JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2848                 MacroAssembler::Zero,
2849                 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2850             speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2851             notWasteful.link(&m_jit);
2852         }
2853     }
2854     return done;
2855 }
2856
2857 void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2858 {
2859     switch (elementSize(type)) {
2860     case 1:
2861         if (isSigned(type))
2862             m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2863         else
2864             m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2865         break;
2866     case 2:
2867         if (isSigned(type))
2868             m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2869         else
2870             m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2871         break;
2872     case 4:
2873         m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2874         break;
2875     default:
2876         CRASH();
2877     }
2878 }
2879
2880 void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2881 {
2882     if (elementSize(type) < 4 || isSigned(type)) {
2883         int32Result(resultReg, node);
2884         return;
2885     }
2886
2887     ASSERT(elementSize(type) == 4 && !isSigned(type));
2888     if (node->shouldSpeculateInt32() && canSpeculate) {
2889         speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2890         int32Result(resultReg, node);
2891         return;
2892     }
2893
2894 #if USE(JSVALUE64)
2895     if (node->shouldSpeculateAnyInt()) {
2896         m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2897         strictInt52Result(resultReg, node);
2898         return;
2899     }
2900 #endif
2901
2902     FPRTemporary fresult(this);
2903     m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2904     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2905     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2906     positive.link(&m_jit);
2907     doubleResult(fresult.fpr(), node);
2908 }
2909
2910 void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2911 {
2912     ASSERT(isInt(type));
2913
2914     SpeculateCellOperand base(this, node->child1());
2915     SpeculateStrictInt32Operand property(this, node->child2());
2916     StorageOperand storage(this, node->child3());
2917
2918     GPRReg baseReg = base.gpr();
2919     GPRReg propertyReg = property.gpr();
2920     GPRReg storageReg = storage.gpr();
2921
2922     GPRTemporary result(this);
2923     GPRReg resultReg = result.gpr();
2924
2925     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
2926
2927     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2928     loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2929     bool canSpeculate = true;
2930     setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2931 }
2932
2933 bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2934     GPRTemporary& value,
2935     GPRReg property,
2936 #if USE(JSVALUE32_64)
2937     GPRTemporary& propertyTag,
2938     GPRTemporary& valueTag,
2939 #endif
2940     Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2941 {
2942     bool isAppropriateConstant = false;
2943     if (valueUse->isConstant()) {
2944         JSValue jsValue = valueUse->asJSValue();
2945         SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2946         SpeculatedType actualType = speculationFromValue(jsValue);
2947         isAppropriateConstant = (expectedType | actualType) == expectedType;
2948     }
2949
2950     if (isAppropriateConstant) {
2951         JSValue jsValue = valueUse->asJSValue();
2952         if (!jsValue.isNumber()) {
2953             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2954             return false;
2955         }
2956         double d = jsValue.asNumber();
2957         if (isClamped)
2958             d = clampDoubleToByte(d);
2959         GPRTemporary scratch(this);
2960         GPRReg scratchReg = scratch.gpr();
2961         m_jit.move(Imm32(toInt32(d)), scratchReg);
2962         value.adopt(scratch);
2963     } else {
2964         switch (valueUse.useKind()) {
2965         case Int32Use: {
2966             SpeculateInt32Operand valueOp(this, valueUse);
2967             GPRTemporary scratch(this);
2968             GPRReg scratchReg = scratch.gpr();
2969             m_jit.move(valueOp.gpr(), scratchReg);
2970             if (isClamped)
2971                 compileClampIntegerToByte(m_jit, scratchReg);
2972             value.adopt(scratch);
2973             break;
2974         }
2975
2976 #if USE(JSVALUE64)
2977         case Int52RepUse: {
2978             SpeculateStrictInt52Operand valueOp(this, valueUse);
2979             GPRTemporary scratch(this);
2980             GPRReg scratchReg = scratch.gpr();
2981             m_jit.move(valueOp.gpr(), scratchReg);
2982             if (isClamped) {
2983                 MacroAssembler::Jump inBounds = m_jit.branch64(
2984                     MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
2985                 MacroAssembler::Jump tooBig = m_jit.branch64(
2986                     MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
2987                 m_jit.move(TrustedImm32(0), scratchReg);
2988                 MacroAssembler::Jump clamped = m_jit.jump();
2989                 tooBig.link(&m_jit);
2990                 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
2991                 clamped.link(&m_jit);
2992                 inBounds.link(&m_jit);
2993             }
2994             value.adopt(scratch);
2995             break;
2996         }
2997 #endif // USE(JSVALUE64)
2998
2999         case DoubleRepUse: {
3000             RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
3001             if (isClamped) {
3002                 SpeculateDoubleOperand valueOp(this, valueUse);
3003                 GPRTemporary result(this);
3004                 FPRTemporary floatScratch(this);
3005                 FPRReg fpr = valueOp.fpr();
3006                 GPRReg gpr = result.gpr();
3007                 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
3008                 value.adopt(result);
3009             } else {
3010 #if USE(JSVALUE32_64)
3011                 GPRTemporary realPropertyTag(this);
3012                 propertyTag.adopt(realPropertyTag);
3013                 GPRReg propertyTagGPR = propertyTag.gpr();
3014
3015                 GPRTemporary realValueTag(this);
3016                 valueTag.adopt(realValueTag);
3017                 GPRReg valueTagGPR = valueTag.gpr();
3018 #endif
3019                 SpeculateDoubleOperand valueOp(this, valueUse);
3020                 GPRTemporary result(this);
3021                 FPRReg fpr = valueOp.fpr();
3022                 GPRReg gpr = result.gpr();
3023                 MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr);
3024                 m_jit.xorPtr(gpr, gpr);
3025                 MacroAssembler::JumpList fixed(m_jit.jump());
3026                 notNaN.link(&m_jit);
3027
3028                 fixed.append(m_jit.branchTruncateDoubleToInt32(
3029                     fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3030
3031 #if USE(JSVALUE64)
3032                 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3033                 boxDouble(fpr, gpr);
3034 #else
3035                 UNUSED_PARAM(property);
3036                 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3037                 boxDouble(fpr, valueTagGPR, gpr);
3038 #endif
3039                 slowPathCases.append(m_jit.jump());
3040
3041                 fixed.link(&m_jit);
3042                 value.adopt(result);
3043             }
3044             break;
3045         }
3046
3047         default:
3048             RELEASE_ASSERT_NOT_REACHED();
3049             break;
3050         }
3051     }
3052     return true;
3053 }
3054
3055 void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3056 {
3057     ASSERT(isInt(type));
3058
3059     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3060     GPRReg storageReg = storage.gpr();
3061
3062     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3063
3064     GPRTemporary value;
3065 #if USE(JSVALUE32_64)
3066     GPRTemporary propertyTag;
3067     GPRTemporary valueTag;
3068 #endif
3069
3070     JITCompiler::JumpList slowPathCases;
3071
3072     bool result = getIntTypedArrayStoreOperand(
3073         value, property,
3074 #if USE(JSVALUE32_64)
3075         propertyTag, valueTag,
3076 #endif
3077         valueUse, slowPathCases, isClamped(type));
3078     if (!result) {
3079         noResult(node);
3080         return;
3081     }
3082
3083     GPRReg valueGPR = value.gpr();
3084 #if USE(JSVALUE32_64)
3085     GPRReg propertyTagGPR = propertyTag.gpr();
3086     GPRReg valueTagGPR = valueTag.gpr();
3087 #endif
3088
3089     ASSERT_UNUSED(valueGPR, valueGPR != property);
3090     ASSERT(valueGPR != base);
3091     ASSERT(valueGPR != storageReg);
3092     JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3093
3094     switch (elementSize(type)) {
3095     case 1:
3096         m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3097         break;
3098     case 2:
3099         m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3100         break;
3101     case 4:
3102         m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3103         break;
3104     default:
3105         CRASH();
3106     }
3107
3108     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3109     if (done.isSet())
3110         done.link(&m_jit);
3111
3112     if (!slowPathCases.empty()) {
3113 #if USE(JSVALUE64)
3114         if (node->op() == PutByValDirect) {
3115             addSlowPathGenerator(slowPathCall(
3116                 slowPathCases, this,
3117                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3118                 NoResult, base, property, valueGPR));
3119         } else {
3120             addSlowPathGenerator(slowPathCall(
3121                 slowPathCases, this,
3122                 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3123                 NoResult, base, property, valueGPR));
3124         }
3125 #else // not USE(JSVALUE64)
3126         if (node->op() == PutByValDirect) {
3127             addSlowPathGenerator(slowPathCall(
3128                 slowPathCases, this,
3129                 m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3130                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3131         } else {
3132             addSlowPathGenerator(slowPathCall(
3133                 slowPathCases, this,
3134                 m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3135                 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3136         }
3137 #endif
3138     }
3139
3140     noResult(node);
3141 }
3142
3143 void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3144 {
3145     ASSERT(isFloat(type));
3146
3147     SpeculateCellOperand base(this, node->child1());
3148     SpeculateStrictInt32Operand property(this, node->child2());
3149     StorageOperand storage(this, node->child3());
3150
3151     GPRReg baseReg = base.gpr();
3152     GPRReg propertyReg = property.gpr();
3153     GPRReg storageReg = storage.gpr();
3154
3155     ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
3156
3157     FPRTemporary result(this);
3158     FPRReg resultReg = result.fpr();
3159     emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3160     switch (elementSize(type)) {
3161     case 4:
3162         m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3163         m_jit.convertFloatToDouble(resultReg, resultReg);
3164         break;
3165     case 8: {
3166         m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3167         break;
3168     }
3169     default:
3170         RELEASE_ASSERT_NOT_REACHED();
3171     }
3172
3173     doubleResult(resultReg, node);
3174 }
3175
3176 void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3177 {
3178     ASSERT(isFloat(type));
3179
3180     StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3181     GPRReg storageReg = storage.gpr();
3182
3183     Edge baseUse = m_jit.graph().varArgChild(node, 0);
3184     Edge valueUse = m_jit.graph().varArgChild(node, 2);
3185
3186     SpeculateDoubleOperand valueOp(this, valueUse);
3187     FPRTemporary scratch(this);
3188     FPRReg valueFPR = valueOp.fpr();
3189     FPRReg scratchFPR = scratch.fpr();
3190
3191     ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3192
3193     MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3194
3195     switch (elementSize(type)) {
3196     case 4: {
3197         m_jit.moveDouble(valueFPR, scratchFPR);
3198         m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3199         m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3200         break;
3201     }
3202     case 8:
3203         m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3204         break;
3205     default:
3206         RELEASE_ASSERT_NOT_REACHED();
3207     }
3208
3209     JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3210     if (done.isSet())
3211         done.link(&m_jit);
3212     noResult(node);
3213 }
3214
3215 void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3216 {
3217     SpeculateCellOperand arg1(this, node->child1());
3218     SpeculateCellOperand arg2(this, node->child2());
3219
3220     GPRReg arg1GPR = arg1.gpr();
3221     GPRReg arg2GPR = arg2.gpr();
3222
3223     speculateObject(node->child1(), arg1GPR);
3224     speculateString(node->child2(), arg2GPR);
3225
3226     flushRegisters();
3227     JSValueRegsFlushedCallResult result(this);
3228     JSValueRegs resultRegs = result.regs();
3229     callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3230     m_jit.exceptionCheck();
3231
3232     jsValueResult(resultRegs, node);
3233 }
3234
3235 void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3236 {
3237     SpeculateCellOperand arg1(this, node->child1());
3238     SpeculateCellOperand arg2(this, node->child2());
3239
3240     GPRReg arg1GPR = arg1.gpr();
3241     GPRReg arg2GPR = arg2.gpr();
3242
3243     speculateObject(node->child1(), arg1GPR);
3244     speculateSymbol(node->child2(), arg2GPR);
3245
3246     flushRegisters();
3247     JSValueRegsFlushedCallResult result(this);
3248     JSValueRegs resultRegs = result.regs();
3249     callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3250     m_jit.exceptionCheck();
3251
3252     jsValueResult(resultRegs, node);
3253 }
3254
3255 void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3256 {
3257     SpeculateCellOperand arg1(this, child1);
3258     SpeculateCellOperand arg2(this, child2);
3259     JSValueOperand arg3(this, child3);
3260
3261     GPRReg arg1GPR = arg1.gpr();
3262     GPRReg arg2GPR = arg2.gpr();
3263     JSValueRegs arg3Regs = arg3.jsValueRegs();
3264
3265     speculateString(child2, arg2GPR);
3266
3267     flushRegisters();
3268     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3269     m_jit.exceptionCheck();
3270
3271     noResult(node);
3272 }
3273
3274 void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3275 {
3276     SpeculateCellOperand arg1(this, child1);
3277     SpeculateCellOperand arg2(this, child2);
3278     JSValueOperand arg3(this, child3);
3279
3280     GPRReg arg1GPR = arg1.gpr();
3281     GPRReg arg2GPR = arg2.gpr();
3282     JSValueRegs arg3Regs = arg3.jsValueRegs();
3283
3284     speculateSymbol(child2, arg2GPR);
3285
3286     flushRegisters();
3287     callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3288     m_jit.exceptionCheck();
3289
3290     noResult(node);
3291 }
3292
3293 void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg, GPRReg scratch2Reg, GPRReg scratch3Reg)
3294 {
3295     // Check that prototype is an object.
3296     speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(prototypeReg));
3297
3298     // Initialize scratchReg with the value being checked.
3299     m_jit.move(valueReg, scratchReg);
3300
3301     // Walk up the prototype chain of the value (in scratchReg), comparing to prototypeReg.
3302     MacroAssembler::Label loop(&m_jit);
3303     MacroAssembler::Jump performDefaultHasInstance = m_jit.branch8(MacroAssembler::Equal,
3304         MacroAssembler::Address(scratchReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ProxyObjectType));
3305     m_jit.emitLoadStructure(*m_jit.vm(), scratchReg, scratch3Reg, scratch2Reg);
3306 #if USE(JSVALUE64)
3307     m_jit.load64(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset()), scratch3Reg);
3308     auto hasMonoProto = m_jit.branchTest64(JITCompiler::NonZero, scratch3Reg);
3309     m_jit.load64(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset)), scratch3Reg);
3310     hasMonoProto.link(&m_jit);
3311     m_jit.move(scratch3Reg, scratchReg);
3312 #else
3313     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + TagOffset), scratch2Reg);
3314     m_jit.load32(MacroAssembler::Address(scratch3Reg, Structure::prototypeOffset() + PayloadOffset), scratch3Reg);
3315     auto hasMonoProto = m_jit.branch32(CCallHelpers::NotEqual, scratch2Reg, TrustedImm32(JSValue::EmptyValueTag));
3316     m_jit.load32(JITCompiler::Address(scratchReg, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), scratch3Reg);
3317     hasMonoProto.link(&m_jit);
3318     m_jit.move(scratch3Reg, scratchReg);
3319 #endif
3320
3321     MacroAssembler::Jump isInstance = m_jit.branchPtr(MacroAssembler::Equal, scratchReg, prototypeReg);
3322 #if USE(JSVALUE64)
3323     m_jit.branchIfCell(JSValueRegs(scratchReg)).linkTo(loop, &m_jit);
3324 #else
3325     m_jit.branchTestPtr(MacroAssembler::NonZero, scratchReg).linkTo(loop, &m_jit);
3326 #endif
3327
3328     // No match - result is false.
3329 #if USE(JSVALUE64)
3330     m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), scratchReg);
3331 #else
3332     m_jit.move(MacroAssembler::TrustedImm32(0), scratchReg);
3333 #endif
3334     MacroAssembler::JumpList doneJumps;
3335     doneJumps.append(m_jit.jump());
3336
3337     performDefaultHasInstance.link(&m_jit);
3338     silentSpillAllRegisters(scratchReg);
3339     callOperation(operationDefaultHasInstance, scratchReg, valueReg, prototypeReg);
3340     silentFillAllRegisters();
3341     m_jit.exceptionCheck();
3342 #if USE(JSVALUE64)
3343     m_jit.or32(TrustedImm32(ValueFalse), scratchReg);
3344 #endif
3345     doneJumps.append(m_jit.jump());
3346