Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp

   1 /*
   2  * Copyright (C) 2011, 2012, 2013 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "DFGSpeculativeJIT.h"
  28
  29 #if ENABLE(DFG_JIT)
  30
  31 #include "Arguments.h"
  32 #include "ArrayPrototype.h"
  33 #include "DFGAbstractInterpreterInlines.h"
  34 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
  35 #include "DFGSlowPathGenerator.h"
  36 #include "JSCJSValueInlines.h"
  37 #include "ObjectPrototype.h"
  38
  39 namespace JSC { namespace DFG {
  40
  41 #if USE(JSVALUE64)
  42
  43 GPRReg SpeculativeJIT::fillJSValue(Edge edge)
  44 {
  45     VirtualRegister virtualRegister = edge->virtualRegister();
  46     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
  47
  48     switch (info.registerFormat()) {
  49     case DataFormatNone: {
  50         GPRReg gpr = allocate();
  51
  52         if (edge->hasConstant()) {
  53             if (isInt32Constant(edge.node())) {
  54                 info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
  55                 JSValue jsValue = jsNumber(valueOfInt32Constant(edge.node()));
  56                 m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr);
  57             } else if (isNumberConstant(edge.node())) {
  58                 info.fillJSValue(*m_stream, gpr, DataFormatJSDouble);
  59                 JSValue jsValue(JSValue::EncodeAsDouble, valueOfNumberConstant(edge.node()));
  60                 m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr);
  61             } else {
  62                 ASSERT(isJSConstant(edge.node()));
  63                 JSValue jsValue = valueOfJSConstant(edge.node());
  64                 m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
  65                 info.fillJSValue(*m_stream, gpr, DataFormatJS);
  66             }
  67
  68             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
  69         } else {
  70             DataFormat spillFormat = info.spillFormat();
  71             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
  72             if (spillFormat == DataFormatInt32) {
  73                 m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
  74                 m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
  75                 spillFormat = DataFormatJSInt32;
  76             } else {
  77                 m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
  78                 if (spillFormat == DataFormatDouble) {
  79                     // Need to box the double, since we want a JSValue.
  80                     m_jit.sub64(GPRInfo::tagTypeNumberRegister, gpr);
  81                     spillFormat = DataFormatJSDouble;
  82                 } else
  83                     RELEASE_ASSERT(spillFormat & DataFormatJS);
  84             }
  85             info.fillJSValue(*m_stream, gpr, spillFormat);
  86         }
  87         return gpr;
  88     }
  89
  90     case DataFormatInt32: {
  91         GPRReg gpr = info.gpr();
  92         // If the register has already been locked we need to take a copy.
  93         // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32.
  94         if (m_gprs.isLocked(gpr)) {
  95             GPRReg result = allocate();
  96             m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr, result);
  97             return result;
  98         }
  99         m_gprs.lock(gpr);
 100         m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
 101         info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
 102         return gpr;
 103     }
 104
 105     case DataFormatDouble: {
 106         FPRReg fpr = info.fpr();
 107         GPRReg gpr = boxDouble(fpr);
 108
 109         // Update all info
 110         info.fillJSValue(*m_stream, gpr, DataFormatJSDouble);
 111         m_fprs.release(fpr);
 112         m_gprs.retain(gpr, virtualRegister, SpillOrderJS);
 113
 114         return gpr;
 115     }
 116
 117     case DataFormatCell:
 118         // No retag required on JSVALUE64!
 119     case DataFormatJS:
 120     case DataFormatJSInt32:
 121     case DataFormatJSDouble:
 122     case DataFormatJSCell:
 123     case DataFormatJSBoolean: {
 124         GPRReg gpr = info.gpr();
 125         m_gprs.lock(gpr);
 126         return gpr;
 127     }
 128
 129     case DataFormatBoolean:
 130     case DataFormatStorage:
 131         // this type currently never occurs
 132         RELEASE_ASSERT_NOT_REACHED();
 133
 134     default:
 135         RELEASE_ASSERT_NOT_REACHED();
 136         return InvalidGPRReg;
 137     }
 138 }
 139
 140 void SpeculativeJIT::nonSpeculativeUInt32ToNumber(Node* node)
 141 {
 142     SpeculateInt32Operand op1(this, node->child1());
 143     FPRTemporary boxer(this);
 144     GPRTemporary result(this, Reuse, op1);
 145
 146     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, op1.gpr(), TrustedImm32(0));
 147
 148     m_jit.convertInt32ToDouble(op1.gpr(), boxer.fpr());
 149     m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), boxer.fpr());
 150
 151     boxDouble(boxer.fpr(), result.gpr());
 152
 153     JITCompiler::Jump done = m_jit.jump();
 154
 155     positive.link(&m_jit);
 156
 157     m_jit.or64(GPRInfo::tagTypeNumberRegister, op1.gpr(), result.gpr());
 158
 159     done.link(&m_jit);
 160
 161     jsValueResult(result.gpr(), m_currentNode);
 162 }
 163
 164 void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
 165 {
 166     JITCompiler::DataLabelPtr structureToCompare;
 167     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(baseGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 168
 169     JITCompiler::ConvertibleLoadLabel propertyStorageLoad =
 170         m_jit.convertibleLoadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
 171     JITCompiler::DataLabelCompact loadWithPatch = m_jit.load64WithCompactAddressOffsetPatch(JITCompiler::Address(resultGPR, 0), resultGPR);
 172
 173     JITCompiler::Label doneLabel = m_jit.label();
 174
 175     OwnPtr<SlowPathGenerator> slowPath;
 176     if (!slowPathTarget.isSet()) {
 177         slowPath = slowPathCall(
 178             structureCheck.m_jump, this, operationGetByIdOptimize, resultGPR, baseGPR,
 179             identifierUID(identifierNumber), spillMode);
 180     } else {
 181         JITCompiler::JumpList slowCases;
 182         slowCases.append(structureCheck.m_jump);
 183         slowCases.append(slowPathTarget);
 184         slowPath = slowPathCall(
 185             slowCases, this, operationGetByIdOptimize, resultGPR, baseGPR,
 186             identifierUID(identifierNumber), spillMode);
 187     }
 188     m_jit.addPropertyAccess(
 189         PropertyAccessRecord(
 190             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad, loadWithPatch,
 191             slowPath.get(), doneLabel, safeCast<int8_t>(baseGPR), safeCast<int8_t>(resultGPR),
 192             usedRegisters(),
 193             spillMode == NeedToSpill ? PropertyAccessRecord::RegistersInUse : PropertyAccessRecord::RegistersFlushed));
 194     addSlowPathGenerator(slowPath.release());
 195 }
 196
 197 void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, Edge valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget)
 198 {
 199
 200     JITCompiler::DataLabelPtr structureToCompare;
 201     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(baseGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 202
 203     writeBarrier(baseGPR, valueGPR, valueUse, WriteBarrierForPropertyAccess, scratchGPR);
 204
 205     JITCompiler::ConvertibleLoadLabel propertyStorageLoad =
 206         m_jit.convertibleLoadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);
 207     JITCompiler::DataLabel32 storeWithPatch = m_jit.store64WithAddressOffsetPatch(valueGPR, JITCompiler::Address(scratchGPR, 0));
 208
 209     JITCompiler::Label doneLabel = m_jit.label();
 210
 211     V_DFGOperation_EJCI optimizedCall;
 212     if (m_jit.strictModeFor(m_currentNode->codeOrigin)) {
 213         if (putKind == Direct)
 214             optimizedCall = operationPutByIdDirectStrictOptimize;
 215         else
 216             optimizedCall = operationPutByIdStrictOptimize;
 217     } else {
 218         if (putKind == Direct)
 219             optimizedCall = operationPutByIdDirectNonStrictOptimize;
 220         else
 221             optimizedCall = operationPutByIdNonStrictOptimize;
 222     }
 223     OwnPtr<SlowPathGenerator> slowPath;
 224     if (!slowPathTarget.isSet()) {
 225         slowPath = slowPathCall(
 226             structureCheck.m_jump, this, optimizedCall, NoResult, valueGPR, baseGPR,
 227             identifierUID(identifierNumber));
 228     } else {
 229         JITCompiler::JumpList slowCases;
 230         slowCases.append(structureCheck.m_jump);
 231         slowCases.append(slowPathTarget);
 232         slowPath = slowPathCall(
 233             slowCases, this, optimizedCall, NoResult, valueGPR, baseGPR,
 234             identifierUID(identifierNumber));
 235     }
 236     RegisterSet currentlyUsedRegisters = usedRegisters();
 237     currentlyUsedRegisters.clear(scratchGPR);
 238     ASSERT(currentlyUsedRegisters.get(baseGPR));
 239     ASSERT(currentlyUsedRegisters.get(valueGPR));
 240     m_jit.addPropertyAccess(
 241         PropertyAccessRecord(
 242             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad,
 243             JITCompiler::DataLabelCompact(storeWithPatch.label()), slowPath.get(), doneLabel,
 244             safeCast<int8_t>(baseGPR), safeCast<int8_t>(valueGPR), currentlyUsedRegisters));
 245     addSlowPathGenerator(slowPath.release());
 246 }
 247
 248 void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert)
 249 {
 250     JSValueOperand arg(this, operand);
 251     GPRReg argGPR = arg.gpr();
 252
 253     GPRTemporary result(this, Reuse, arg);
 254     GPRReg resultGPR = result.gpr();
 255
 256     JITCompiler::Jump notCell;
 257
 258     JITCompiler::Jump notMasqueradesAsUndefined;
 259     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
 260         if (!isKnownCell(operand.node()))
 261             notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister);
 262
 263         speculationWatchpointForMasqueradesAsUndefined();
 264
 265         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR);
 266         notMasqueradesAsUndefined = m_jit.jump();
 267     } else {
 268         GPRTemporary localGlobalObject(this);
 269         GPRTemporary remoteGlobalObject(this);
 270
 271         if (!isKnownCell(operand.node()))
 272             notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister);
 273
 274         m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR);
 275         JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined));
 276
 277         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR);
 278         notMasqueradesAsUndefined = m_jit.jump();
 279
 280         isMasqueradesAsUndefined.link(&m_jit);
 281         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 282         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 283         m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR);
 284         m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 285         m_jit.comparePtr(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultGPR);
 286     }
 287
 288     if (!isKnownCell(operand.node())) {
 289         JITCompiler::Jump done = m_jit.jump();
 290
 291         notCell.link(&m_jit);
 292
 293         m_jit.move(argGPR, resultGPR);
 294         m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR);
 295         m_jit.compare64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(ValueNull), resultGPR);
 296
 297         done.link(&m_jit);
 298     }
 299
 300     notMasqueradesAsUndefined.link(&m_jit);
 301
 302     m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
 303     jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
 304 }
 305
 306 void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert)
 307 {
 308     BasicBlock* taken = branchNode->takenBlock();
 309     BasicBlock* notTaken = branchNode->notTakenBlock();
 310
 311     if (taken == nextBlock()) {
 312         invert = !invert;
 313         BasicBlock* tmp = taken;
 314         taken = notTaken;
 315         notTaken = tmp;
 316     }
 317
 318     JSValueOperand arg(this, operand);
 319     GPRReg argGPR = arg.gpr();
 320
 321     GPRTemporary result(this, Reuse, arg);
 322     GPRReg resultGPR = result.gpr();
 323
 324     JITCompiler::Jump notCell;
 325
 326     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
 327         if (!isKnownCell(operand.node()))
 328             notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister);
 329
 330         speculationWatchpointForMasqueradesAsUndefined();
 331
 332         jump(invert ? taken : notTaken, ForceJump);
 333     } else {
 334         GPRTemporary localGlobalObject(this);
 335         GPRTemporary remoteGlobalObject(this);
 336
 337         if (!isKnownCell(operand.node()))
 338             notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister);
 339
 340         m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR);
 341         branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken);
 342
 343         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 344         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 345         m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR);
 346         m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 347         branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken);
 348     }
 349
 350     if (!isKnownCell(operand.node())) {
 351         jump(notTaken, ForceJump);
 352
 353         notCell.link(&m_jit);
 354
 355         m_jit.move(argGPR, resultGPR);
 356         m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR);
 357         branch64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm64(ValueNull), taken);
 358     }
 359
 360     jump(notTaken);
 361 }
 362
 363 bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert)
 364 {
 365     unsigned branchIndexInBlock = detectPeepHoleBranch();
 366     if (branchIndexInBlock != UINT_MAX) {
 367         Node* branchNode = m_block->at(branchIndexInBlock);
 368
 369         RELEASE_ASSERT(node->adjustedRefCount() == 1);
 370
 371         nonSpeculativePeepholeBranchNull(operand, branchNode, invert);
 372
 373         use(node->child1());
 374         use(node->child2());
 375         m_indexInBlock = branchIndexInBlock;
 376         m_currentNode = branchNode;
 377
 378         return true;
 379     }
 380
 381     nonSpeculativeNonPeepholeCompareNull(operand, invert);
 382
 383     return false;
 384 }
 385
 386 void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 387 {
 388     BasicBlock* taken = branchNode->takenBlock();
 389     BasicBlock* notTaken = branchNode->notTakenBlock();
 390
 391     JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero;
 392
 393     // The branch instruction will branch to the taken block.
 394     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 395     if (taken == nextBlock()) {
 396         cond = JITCompiler::invert(cond);
 397         callResultCondition = JITCompiler::Zero;
 398         BasicBlock* tmp = taken;
 399         taken = notTaken;
 400         notTaken = tmp;
 401     }
 402
 403     JSValueOperand arg1(this, node->child1());
 404     JSValueOperand arg2(this, node->child2());
 405     GPRReg arg1GPR = arg1.gpr();
 406     GPRReg arg2GPR = arg2.gpr();
 407
 408     JITCompiler::JumpList slowPath;
 409
 410     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 411         GPRResult result(this);
 412         GPRReg resultGPR = result.gpr();
 413
 414         arg1.use();
 415         arg2.use();
 416
 417         flushRegisters();
 418         callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
 419
 420         branchTest32(callResultCondition, resultGPR, taken);
 421     } else {
 422         GPRTemporary result(this, Reuse, arg2);
 423         GPRReg resultGPR = result.gpr();
 424
 425         arg1.use();
 426         arg2.use();
 427
 428         if (!isKnownInteger(node->child1().node()))
 429             slowPath.append(m_jit.branch64(MacroAssembler::Below, arg1GPR, GPRInfo::tagTypeNumberRegister));
 430         if (!isKnownInteger(node->child2().node()))
 431             slowPath.append(m_jit.branch64(MacroAssembler::Below, arg2GPR, GPRInfo::tagTypeNumberRegister));
 432
 433         branch32(cond, arg1GPR, arg2GPR, taken);
 434
 435         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 436             jump(notTaken, ForceJump);
 437
 438             slowPath.link(&m_jit);
 439
 440             silentSpillAllRegisters(resultGPR);
 441             callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
 442             silentFillAllRegisters(resultGPR);
 443
 444             branchTest32(callResultCondition, resultGPR, taken);
 445         }
 446     }
 447
 448     jump(notTaken);
 449
 450     m_indexInBlock = m_block->size() - 1;
 451     m_currentNode = branchNode;
 452 }
 453
 454 template<typename JumpType>
 455 class CompareAndBoxBooleanSlowPathGenerator
 456     : public CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg> {
 457 public:
 458     CompareAndBoxBooleanSlowPathGenerator(
 459         JumpType from, SpeculativeJIT* jit,
 460         S_DFGOperation_EJJ function, GPRReg result, GPRReg arg1, GPRReg arg2)
 461         : CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg>(
 462             from, jit, function, NeedToSpill, result)
 463         , m_arg1(arg1)
 464         , m_arg2(arg2)
 465     {
 466     }
 467
 468 protected:
 469     virtual void generateInternal(SpeculativeJIT* jit)
 470     {
 471         this->setUp(jit);
 472         this->recordCall(jit->callOperation(this->m_function, this->m_result, m_arg1, m_arg2));
 473         jit->m_jit.and32(JITCompiler::TrustedImm32(1), this->m_result);
 474         jit->m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), this->m_result);
 475         this->tearDown(jit);
 476     }
 477
 478 private:
 479     GPRReg m_arg1;
 480     GPRReg m_arg2;
 481 };
 482
 483 void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 484 {
 485     JSValueOperand arg1(this, node->child1());
 486     JSValueOperand arg2(this, node->child2());
 487     GPRReg arg1GPR = arg1.gpr();
 488     GPRReg arg2GPR = arg2.gpr();
 489
 490     JITCompiler::JumpList slowPath;
 491
 492     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 493         GPRResult result(this);
 494         GPRReg resultGPR = result.gpr();
 495
 496         arg1.use();
 497         arg2.use();
 498
 499         flushRegisters();
 500         callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
 501
 502         m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
 503         jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
 504     } else {
 505         GPRTemporary result(this, Reuse, arg2);
 506         GPRReg resultGPR = result.gpr();
 507
 508         arg1.use();
 509         arg2.use();
 510
 511         if (!isKnownInteger(node->child1().node()))
 512             slowPath.append(m_jit.branch64(MacroAssembler::Below, arg1GPR, GPRInfo::tagTypeNumberRegister));
 513         if (!isKnownInteger(node->child2().node()))
 514             slowPath.append(m_jit.branch64(MacroAssembler::Below, arg2GPR, GPRInfo::tagTypeNumberRegister));
 515
 516         m_jit.compare32(cond, arg1GPR, arg2GPR, resultGPR);
 517         m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
 518
 519         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 520             addSlowPathGenerator(adoptPtr(
 521                 new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>(
 522                     slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR)));
 523         }
 524
 525         jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
 526     }
 527 }
 528
 529 void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert)
 530 {
 531     BasicBlock* taken = branchNode->takenBlock();
 532     BasicBlock* notTaken = branchNode->notTakenBlock();
 533
 534     // The branch instruction will branch to the taken block.
 535     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 536     if (taken == nextBlock()) {
 537         invert = !invert;
 538         BasicBlock* tmp = taken;
 539         taken = notTaken;
 540         notTaken = tmp;
 541     }
 542
 543     JSValueOperand arg1(this, node->child1());
 544     JSValueOperand arg2(this, node->child2());
 545     GPRReg arg1GPR = arg1.gpr();
 546     GPRReg arg2GPR = arg2.gpr();
 547
 548     GPRTemporary result(this);
 549     GPRReg resultGPR = result.gpr();
 550
 551     arg1.use();
 552     arg2.use();
 553
 554     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 555         // see if we get lucky: if the arguments are cells and they reference the same
 556         // cell, then they must be strictly equal.
 557         branch64(JITCompiler::Equal, arg1GPR, arg2GPR, invert ? notTaken : taken);
 558
 559         silentSpillAllRegisters(resultGPR);
 560         callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR);
 561         silentFillAllRegisters(resultGPR);
 562
 563         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken);
 564     } else {
 565         m_jit.or64(arg1GPR, arg2GPR, resultGPR);
 566
 567         JITCompiler::Jump twoCellsCase = m_jit.branchTest64(JITCompiler::Zero, resultGPR, GPRInfo::tagMaskRegister);
 568
 569         JITCompiler::Jump leftOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg1GPR, GPRInfo::tagTypeNumberRegister);
 570         JITCompiler::Jump leftDouble = m_jit.branchTest64(JITCompiler::NonZero, arg1GPR, GPRInfo::tagTypeNumberRegister);
 571         leftOK.link(&m_jit);
 572         JITCompiler::Jump rightOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg2GPR, GPRInfo::tagTypeNumberRegister);
 573         JITCompiler::Jump rightDouble = m_jit.branchTest64(JITCompiler::NonZero, arg2GPR, GPRInfo::tagTypeNumberRegister);
 574         rightOK.link(&m_jit);
 575
 576         branch64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, arg1GPR, arg2GPR, taken);
 577         jump(notTaken, ForceJump);
 578
 579         twoCellsCase.link(&m_jit);
 580         branch64(JITCompiler::Equal, arg1GPR, arg2GPR, invert ? notTaken : taken);
 581
 582         leftDouble.link(&m_jit);
 583         rightDouble.link(&m_jit);
 584
 585         silentSpillAllRegisters(resultGPR);
 586         callOperation(operationCompareStrictEq, resultGPR, arg1GPR, arg2GPR);
 587         silentFillAllRegisters(resultGPR);
 588
 589         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken);
 590     }
 591
 592     jump(notTaken);
 593 }
 594
 595 void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert)
 596 {
 597     JSValueOperand arg1(this, node->child1());
 598     JSValueOperand arg2(this, node->child2());
 599     GPRReg arg1GPR = arg1.gpr();
 600     GPRReg arg2GPR = arg2.gpr();
 601
 602     GPRTemporary result(this);
 603     GPRReg resultGPR = result.gpr();
 604
 605     arg1.use();
 606     arg2.use();
 607
 608     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 609         // see if we get lucky: if the arguments are cells and they reference the same
 610         // cell, then they must be strictly equal.
 611         // FIXME: this should flush registers instead of silent spill/fill.
 612         JITCompiler::Jump notEqualCase = m_jit.branch64(JITCompiler::NotEqual, arg1GPR, arg2GPR);
 613
 614         m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR);
 615
 616         JITCompiler::Jump done = m_jit.jump();
 617
 618         notEqualCase.link(&m_jit);
 619
 620         silentSpillAllRegisters(resultGPR);
 621         callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR);
 622         silentFillAllRegisters(resultGPR);
 623
 624         m_jit.and64(JITCompiler::TrustedImm32(1), resultGPR);
 625         m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), resultGPR);
 626
 627         done.link(&m_jit);
 628     } else {
 629         m_jit.or64(arg1GPR, arg2GPR, resultGPR);
 630
 631         JITCompiler::JumpList slowPathCases;
 632
 633         JITCompiler::Jump twoCellsCase = m_jit.branchTest64(JITCompiler::Zero, resultGPR, GPRInfo::tagMaskRegister);
 634
 635         JITCompiler::Jump leftOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg1GPR, GPRInfo::tagTypeNumberRegister);
 636         slowPathCases.append(m_jit.branchTest64(JITCompiler::NonZero, arg1GPR, GPRInfo::tagTypeNumberRegister));
 637         leftOK.link(&m_jit);
 638         JITCompiler::Jump rightOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg2GPR, GPRInfo::tagTypeNumberRegister);
 639         slowPathCases.append(m_jit.branchTest64(JITCompiler::NonZero, arg2GPR, GPRInfo::tagTypeNumberRegister));
 640         rightOK.link(&m_jit);
 641
 642         m_jit.compare64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, arg1GPR, arg2GPR, resultGPR);
 643         m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), resultGPR);
 644
 645         JITCompiler::Jump done = m_jit.jump();
 646
 647         twoCellsCase.link(&m_jit);
 648         slowPathCases.append(m_jit.branch64(JITCompiler::NotEqual, arg1GPR, arg2GPR));
 649
 650         m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR);
 651
 652         addSlowPathGenerator(
 653             adoptPtr(
 654                 new CompareAndBoxBooleanSlowPathGenerator<MacroAssembler::JumpList>(
 655                     slowPathCases, this, operationCompareStrictEq, resultGPR, arg1GPR,
 656                     arg2GPR)));
 657
 658         done.link(&m_jit);
 659     }
 660
 661     jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
 662 }
 663
 664 void SpeculativeJIT::emitCall(Node* node)
 665 {
 666     if (node->op() != Call)
 667         RELEASE_ASSERT(node->op() == Construct);
 668
 669     // For constructors, the this argument is not passed but we have to make space
 670     // for it.
 671     int dummyThisArgument = node->op() == Call ? 0 : 1;
 672
 673     CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct;
 674
 675     Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()];
 676     JSValueOperand callee(this, calleeEdge);
 677     GPRReg calleeGPR = callee.gpr();
 678     use(calleeEdge);
 679
 680     // The call instruction's first child is the function; the subsequent children are the
 681     // arguments.
 682     int numPassedArgs = node->numChildren() - 1;
 683
 684     m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs + dummyThisArgument), callFramePayloadSlot(JSStack::ArgumentCount));
 685     m_jit.store64(GPRInfo::callFrameRegister, callFrameSlot(JSStack::CallerFrame));
 686     m_jit.store64(calleeGPR, callFrameSlot(JSStack::Callee));
 687
 688     for (int i = 0; i < numPassedArgs; i++) {
 689         Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i];
 690         JSValueOperand arg(this, argEdge);
 691         GPRReg argGPR = arg.gpr();
 692         use(argEdge);
 693
 694         m_jit.store64(argGPR, argumentSlot(i + dummyThisArgument));
 695     }
 696
 697     flushRegisters();
 698
 699     GPRResult result(this);
 700     GPRReg resultGPR = result.gpr();
 701
 702     JITCompiler::DataLabelPtr targetToCheck;
 703     JITCompiler::JumpList slowPath;
 704
 705     CallBeginToken token;
 706     m_jit.beginCall(node->codeOrigin, token);
 707
 708     m_jit.addPtr(TrustedImm32(-(m_jit.codeBlock()->m_numCalleeRegisters * sizeof(Register))), GPRInfo::callFrameRegister);
 709
 710     slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, MacroAssembler::TrustedImmPtr(0)));
 711
 712     m_jit.loadPtr(MacroAssembler::Address(calleeGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultGPR);
 713     m_jit.store64(resultGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain));
 714
 715     CodeOrigin codeOrigin = m_currentNode->codeOrigin;
 716     JITCompiler::Call fastCall = m_jit.nearCall();
 717     m_jit.notifyCall(fastCall, codeOrigin, token);
 718
 719     JITCompiler::Jump done = m_jit.jump();
 720
 721     slowPath.link(&m_jit);
 722
 723     m_jit.move(calleeGPR, GPRInfo::nonArgGPR0);
 724     m_jit.prepareForExceptionCheck();
 725     JITCompiler::Call slowCall = m_jit.nearCall();
 726     m_jit.notifyCall(slowCall, codeOrigin, token);
 727
 728     done.link(&m_jit);
 729
 730     m_jit.move(GPRInfo::returnValueGPR, resultGPR);
 731
 732     jsValueResult(resultGPR, m_currentNode, DataFormatJS, UseChildrenCalledExplicitly);
 733
 734     m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleeGPR, m_currentNode->codeOrigin);
 735 }
 736
 737 template<bool strict>
 738 GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnFormat)
 739 {
 740 #if DFG_ENABLE(DEBUG_VERBOSE)
 741     dataLogF("SpecInt@%d   ", edge->index());
 742 #endif
 743     AbstractValue& value = m_state.forNode(edge);
 744     SpeculatedType type = value.m_type;
 745     ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32));
 746     m_interpreter.filter(value, SpecInt32);
 747     VirtualRegister virtualRegister = edge->virtualRegister();
 748     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 749
 750     switch (info.registerFormat()) {
 751     case DataFormatNone: {
 752         if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) {
 753             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 754             returnFormat = DataFormatInt32;
 755             return allocate();
 756         }
 757
 758         GPRReg gpr = allocate();
 759
 760         if (edge->hasConstant()) {
 761             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
 762             ASSERT(isInt32Constant(edge.node()));
 763             m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 764             info.fillInt32(*m_stream, gpr);
 765             returnFormat = DataFormatInt32;
 766             return gpr;
 767         }
 768
 769         DataFormat spillFormat = info.spillFormat();
 770
 771         RELEASE_ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInt32);
 772
 773         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 774
 775         if (spillFormat == DataFormatJSInt32 || spillFormat == DataFormatInt32) {
 776             // If we know this was spilled as an integer we can fill without checking.
 777             if (strict) {
 778                 m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
 779                 info.fillInt32(*m_stream, gpr);
 780                 returnFormat = DataFormatInt32;
 781                 return gpr;
 782             }
 783             if (spillFormat == DataFormatInt32) {
 784                 m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
 785                 m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
 786             } else
 787                 m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
 788             info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
 789             returnFormat = DataFormatJSInt32;
 790             return gpr;
 791         }
 792         m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
 793
 794         // Fill as JSValue, and fall through.
 795         info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
 796         m_gprs.unlock(gpr);
 797     }
 798
 799     case DataFormatJS: {
 800         // Check the value is an integer.
 801         GPRReg gpr = info.gpr();
 802         m_gprs.lock(gpr);
 803         if (type & ~SpecInt32)
 804             speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branch64(MacroAssembler::Below, gpr, GPRInfo::tagTypeNumberRegister));
 805         info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
 806         // If !strict we're done, return.
 807         if (!strict) {
 808             returnFormat = DataFormatJSInt32;
 809             return gpr;
 810         }
 811         // else fall through & handle as DataFormatJSInt32.
 812         m_gprs.unlock(gpr);
 813     }
 814
 815     case DataFormatJSInt32: {
 816         // In a strict fill we need to strip off the value tag.
 817         if (strict) {
 818             GPRReg gpr = info.gpr();
 819             GPRReg result;
 820             // If the register has already been locked we need to take a copy.
 821             // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32.
 822             if (m_gprs.isLocked(gpr))
 823                 result = allocate();
 824             else {
 825                 m_gprs.lock(gpr);
 826                 info.fillInt32(*m_stream, gpr);
 827                 result = gpr;
 828             }
 829             m_jit.zeroExtend32ToPtr(gpr, result);
 830             returnFormat = DataFormatInt32;
 831             return result;
 832         }
 833
 834         GPRReg gpr = info.gpr();
 835         m_gprs.lock(gpr);
 836         returnFormat = DataFormatJSInt32;
 837         return gpr;
 838     }
 839
 840     case DataFormatInt32: {
 841         GPRReg gpr = info.gpr();
 842         m_gprs.lock(gpr);
 843         returnFormat = DataFormatInt32;
 844         return gpr;
 845     }
 846
 847     case DataFormatDouble:
 848     case DataFormatJSDouble: {
 849         if (edge->hasConstant() && isInt32Constant(edge.node())) {
 850             GPRReg gpr = allocate();
 851             ASSERT(isInt32Constant(edge.node()));
 852             m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 853             returnFormat = DataFormatInt32;
 854             return gpr;
 855         }
 856     }
 857     case DataFormatCell:
 858     case DataFormatBoolean:
 859     case DataFormatJSCell:
 860     case DataFormatJSBoolean: {
 861         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 862         returnFormat = DataFormatInt32;
 863         return allocate();
 864     }
 865
 866     case DataFormatStorage:
 867         RELEASE_ASSERT_NOT_REACHED();
 868
 869     default:
 870         RELEASE_ASSERT_NOT_REACHED();
 871         return InvalidGPRReg;
 872     }
 873 }
 874
 875 GPRReg SpeculativeJIT::fillSpeculateInt32(Edge edge, DataFormat& returnFormat)
 876 {
 877     return fillSpeculateInt32Internal<false>(edge, returnFormat);
 878 }
 879
 880 GPRReg SpeculativeJIT::fillSpeculateInt32Strict(Edge edge)
 881 {
 882     DataFormat mustBeDataFormatInt32;
 883     GPRReg result = fillSpeculateInt32Internal<true>(edge, mustBeDataFormatInt32);
 884     RELEASE_ASSERT(mustBeDataFormatInt32 == DataFormatInt32);
 885     return result;
 886 }
 887
 888 FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge)
 889 {
 890 #if DFG_ENABLE(DEBUG_VERBOSE)
 891     dataLogF("SpecDouble@%d   ", edge->index());
 892 #endif
 893     AbstractValue& value = m_state.forNode(edge);
 894     SpeculatedType type = value.m_type;
 895     ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecNumber));
 896     m_interpreter.filter(value, SpecNumber);
 897     VirtualRegister virtualRegister = edge->virtualRegister();
 898     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 899
 900     if (info.registerFormat() == DataFormatNone) {
 901         if (edge->hasConstant()) {
 902             GPRReg gpr = allocate();
 903
 904             if (isInt32Constant(edge.node())) {
 905                 FPRReg fpr = fprAllocate();
 906                 m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(static_cast<double>(valueOfInt32Constant(edge.node())))), gpr);
 907                 m_jit.move64ToDouble(gpr, fpr);
 908                 unlock(gpr);
 909
 910                 // Don't fill double here since that will lead to confusion: the
 911                 // register allocator will now think that this is a double while
 912                 // everyone else thinks it's an integer.
 913                 return fpr;
 914             }
 915             if (isNumberConstant(edge.node())) {
 916                 FPRReg fpr = fprAllocate();
 917                 m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(edge.node()))), gpr);
 918                 m_jit.move64ToDouble(gpr, fpr);
 919                 unlock(gpr);
 920
 921                 m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
 922                 info.fillDouble(*m_stream, fpr);
 923                 return fpr;
 924             }
 925             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 926             return fprAllocate();
 927         }
 928
 929         DataFormat spillFormat = info.spillFormat();
 930         switch (spillFormat) {
 931         case DataFormatDouble: {
 932             FPRReg fpr = fprAllocate();
 933             m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
 934             m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
 935             info.fillDouble(*m_stream, fpr);
 936             return fpr;
 937         }
 938
 939         case DataFormatInt32: {
 940             GPRReg gpr = allocate();
 941
 942             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 943             m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
 944             info.fillInt32(*m_stream, gpr);
 945             unlock(gpr);
 946             break;
 947         }
 948
 949         default:
 950             GPRReg gpr = allocate();
 951
 952             RELEASE_ASSERT(spillFormat & DataFormatJS);
 953             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 954             m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
 955             info.fillJSValue(*m_stream, gpr, spillFormat);
 956             unlock(gpr);
 957             break;
 958         }
 959     }
 960
 961     switch (info.registerFormat()) {
 962     case DataFormatNone: // Should have filled, above.
 963     case DataFormatBoolean: // This type never occurs.
 964     case DataFormatStorage:
 965         RELEASE_ASSERT_NOT_REACHED();
 966
 967     case DataFormatCell:
 968         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 969         return fprAllocate();
 970
 971     case DataFormatJSCell:
 972     case DataFormatJS:
 973     case DataFormatJSBoolean: {
 974         GPRReg jsValueGpr = info.gpr();
 975         m_gprs.lock(jsValueGpr);
 976         FPRReg fpr = fprAllocate();
 977         GPRReg tempGpr = allocate();
 978
 979         JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, jsValueGpr, GPRInfo::tagTypeNumberRegister);
 980
 981         if (type & ~SpecNumber)
 982             speculationCheck(BadType, JSValueRegs(jsValueGpr), edge, m_jit.branchTest64(MacroAssembler::Zero, jsValueGpr, GPRInfo::tagTypeNumberRegister));
 983
 984         // First, if we get here we have a double encoded as a JSValue
 985         m_jit.move(jsValueGpr, tempGpr);
 986         unboxDouble(tempGpr, fpr);
 987         JITCompiler::Jump hasUnboxedDouble = m_jit.jump();
 988
 989         // Finally, handle integers.
 990         isInteger.link(&m_jit);
 991         m_jit.convertInt32ToDouble(jsValueGpr, fpr);
 992         hasUnboxedDouble.link(&m_jit);
 993
 994         m_gprs.release(jsValueGpr);
 995         m_gprs.unlock(jsValueGpr);
 996         m_gprs.unlock(tempGpr);
 997         m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
 998         info.fillDouble(*m_stream, fpr);
 999         info.killSpilled();
1000         return fpr;
1001     }
1002
1003     case DataFormatJSInt32:
1004     case DataFormatInt32: {
1005         FPRReg fpr = fprAllocate();
1006         GPRReg gpr = info.gpr();
1007         m_gprs.lock(gpr);
1008         m_jit.convertInt32ToDouble(gpr, fpr);
1009         m_gprs.unlock(gpr);
1010         return fpr;
1011     }
1012
1013     // Unbox the double
1014     case DataFormatJSDouble: {
1015         GPRReg gpr = info.gpr();
1016         FPRReg fpr = fprAllocate();
1017         if (m_gprs.isLocked(gpr)) {
1018             // Make sure we don't trample gpr if it is in use.
1019             GPRReg temp = allocate();
1020             m_jit.move(gpr, temp);
1021             unboxDouble(temp, fpr);
1022             unlock(temp);
1023         } else
1024             unboxDouble(gpr, fpr);
1025
1026         m_gprs.release(gpr);
1027         m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
1028
1029         info.fillDouble(*m_stream, fpr);
1030         return fpr;
1031     }
1032
1033     case DataFormatDouble: {
1034         FPRReg fpr = info.fpr();
1035         m_fprs.lock(fpr);
1036         return fpr;
1037     }
1038
1039     default:
1040         RELEASE_ASSERT_NOT_REACHED();
1041         return InvalidFPRReg;
1042     }
1043 }
1044
1045 GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge)
1046 {
1047 #if DFG_ENABLE(DEBUG_VERBOSE)
1048     dataLogF("SpecCell@%d   ", edge->index());
1049 #endif
1050     AbstractValue& value = m_state.forNode(edge);
1051     SpeculatedType type = value.m_type;
1052     ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell));
1053     m_interpreter.filter(value, SpecCell);
1054     VirtualRegister virtualRegister = edge->virtualRegister();
1055     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1056
1057     switch (info.registerFormat()) {
1058     case DataFormatNone: {
1059         if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) {
1060             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1061             return allocate();
1062         }
1063
1064         GPRReg gpr = allocate();
1065
1066         if (edge->hasConstant()) {
1067             JSValue jsValue = valueOfJSConstant(edge.node());
1068             if (jsValue.isCell()) {
1069                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1070                 m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
1071                 info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1072                 return gpr;
1073             }
1074             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1075             return gpr;
1076         }
1077         RELEASE_ASSERT(info.spillFormat() & DataFormatJS);
1078         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1079         m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
1080
1081         info.fillJSValue(*m_stream, gpr, DataFormatJS);
1082         if (type & ~SpecCell)
1083             speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister));
1084         info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1085         return gpr;
1086     }
1087
1088     case DataFormatCell:
1089     case DataFormatJSCell: {
1090         GPRReg gpr = info.gpr();
1091         m_gprs.lock(gpr);
1092 #if DFG_ENABLE(JIT_ASSERT)
1093         MacroAssembler::Jump checkCell = m_jit.branchTest64(MacroAssembler::Zero, gpr, GPRInfo::tagMaskRegister);
1094         m_jit.breakpoint();
1095         checkCell.link(&m_jit);
1096 #endif
1097         return gpr;
1098     }
1099
1100     case DataFormatJS: {
1101         GPRReg gpr = info.gpr();
1102         m_gprs.lock(gpr);
1103         if (type & ~SpecCell)
1104             speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister));
1105         info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1106         return gpr;
1107     }
1108
1109     case DataFormatJSInt32:
1110     case DataFormatInt32:
1111     case DataFormatJSDouble:
1112     case DataFormatDouble:
1113     case DataFormatJSBoolean:
1114     case DataFormatBoolean: {
1115         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1116         return allocate();
1117     }
1118
1119     case DataFormatStorage:
1120         RELEASE_ASSERT_NOT_REACHED();
1121
1122     default:
1123         RELEASE_ASSERT_NOT_REACHED();
1124         return InvalidGPRReg;
1125     }
1126 }
1127
1128 GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge)
1129 {
1130 #if DFG_ENABLE(DEBUG_VERBOSE)
1131     dataLogF("SpecBool@%d   ", edge->index());
1132 #endif
1133     AbstractValue& value = m_state.forNode(edge);
1134     SpeculatedType type = value.m_type;
1135     m_interpreter.filter(value, SpecBoolean);
1136     VirtualRegister virtualRegister = edge->virtualRegister();
1137     GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1138
1139     switch (info.registerFormat()) {
1140     case DataFormatNone: {
1141         if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) {
1142             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1143             return allocate();
1144         }
1145
1146         GPRReg gpr = allocate();
1147
1148         if (edge->hasConstant()) {
1149             JSValue jsValue = valueOfJSConstant(edge.node());
1150             if (jsValue.isBoolean()) {
1151                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1152                 m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
1153                 info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1154                 return gpr;
1155             }
1156             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1157             return gpr;
1158         }
1159         RELEASE_ASSERT(info.spillFormat() & DataFormatJS);
1160         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1161         m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
1162
1163         info.fillJSValue(*m_stream, gpr, DataFormatJS);
1164         if (type & ~SpecBoolean) {
1165             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1166             speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, TrustedImm32(static_cast<int32_t>(~1))), SpeculationRecovery(BooleanSpeculationCheck, gpr, InvalidGPRReg));
1167             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1168         }
1169         info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1170         return gpr;
1171     }
1172
1173     case DataFormatBoolean:
1174     case DataFormatJSBoolean: {
1175         GPRReg gpr = info.gpr();
1176         m_gprs.lock(gpr);
1177         return gpr;
1178     }
1179
1180     case DataFormatJS: {
1181         GPRReg gpr = info.gpr();
1182         m_gprs.lock(gpr);
1183         if (type & ~SpecBoolean) {
1184             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1185             speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, TrustedImm32(static_cast<int32_t>(~1))), SpeculationRecovery(BooleanSpeculationCheck, gpr, InvalidGPRReg));
1186             m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1187         }
1188         info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1189         return gpr;
1190     }
1191
1192     case DataFormatJSInt32:
1193     case DataFormatInt32:
1194     case DataFormatJSDouble:
1195     case DataFormatDouble:
1196     case DataFormatJSCell:
1197     case DataFormatCell: {
1198         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1199         return allocate();
1200     }
1201
1202     case DataFormatStorage:
1203         RELEASE_ASSERT_NOT_REACHED();
1204
1205     default:
1206         RELEASE_ASSERT_NOT_REACHED();
1207         return InvalidGPRReg;
1208     }
1209 }
1210
1211 JITCompiler::Jump SpeculativeJIT::convertToDouble(GPRReg value, FPRReg result, GPRReg tmp)
1212 {
1213     JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, value, GPRInfo::tagTypeNumberRegister);
1214
1215     JITCompiler::Jump notNumber = m_jit.branchTest64(MacroAssembler::Zero, value, GPRInfo::tagTypeNumberRegister);
1216
1217     m_jit.move(value, tmp);
1218     unboxDouble(tmp, result);
1219
1220     JITCompiler::Jump done = m_jit.jump();
1221
1222     isInteger.link(&m_jit);
1223
1224     m_jit.convertInt32ToDouble(value, result);
1225
1226     done.link(&m_jit);
1227
1228     return notNumber;
1229 }
1230
1231 void SpeculativeJIT::compileObjectEquality(Node* node)
1232 {
1233     SpeculateCellOperand op1(this, node->child1());
1234     SpeculateCellOperand op2(this, node->child2());
1235     GPRTemporary result(this, Reuse, op1);
1236
1237     GPRReg op1GPR = op1.gpr();
1238     GPRReg op2GPR = op2.gpr();
1239     GPRReg resultGPR = result.gpr();
1240
1241     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1242         speculationWatchpointForMasqueradesAsUndefined();
1243         DFG_TYPE_CHECK(
1244             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1245                 MacroAssembler::Equal,
1246                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1247                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1248         DFG_TYPE_CHECK(
1249             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1250                 MacroAssembler::Equal,
1251                 MacroAssembler::Address(op2GPR, JSCell::structureOffset()),
1252                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1253     } else {
1254         GPRTemporary structure(this);
1255         GPRReg structureGPR = structure.gpr();
1256
1257         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1258         DFG_TYPE_CHECK(
1259             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1260                 MacroAssembler::Equal,
1261                 structureGPR,
1262                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1263         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1264             m_jit.branchTest8(
1265                 MacroAssembler::NonZero,
1266                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1267                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1268
1269         m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR);
1270         DFG_TYPE_CHECK(
1271             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1272                 MacroAssembler::Equal,
1273                 structureGPR,
1274                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1275         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1276             m_jit.branchTest8(
1277                 MacroAssembler::NonZero,
1278                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1279                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1280     }
1281
1282     MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR);
1283     m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1284     MacroAssembler::Jump done = m_jit.jump();
1285     falseCase.link(&m_jit);
1286     m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1287     done.link(&m_jit);
1288
1289     jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1290 }
1291
1292 void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild)
1293 {
1294     SpeculateCellOperand op1(this, leftChild);
1295     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1296     GPRTemporary result(this);
1297
1298     GPRReg op1GPR = op1.gpr();
1299     GPRReg op2GPR = op2.gpr();
1300     GPRReg resultGPR = result.gpr();
1301     GPRTemporary structure;
1302     GPRReg structureGPR = InvalidGPRReg;
1303
1304     bool masqueradesAsUndefinedWatchpointValid =
1305         masqueradesAsUndefinedWatchpointIsStillValid();
1306
1307     if (!masqueradesAsUndefinedWatchpointValid) {
1308         // The masquerades as undefined case will use the structure register, so allocate it here.
1309         // Do this at the top of the function to avoid branching around a register allocation.
1310         GPRTemporary realStructure(this);
1311         structure.adopt(realStructure);
1312         structureGPR = structure.gpr();
1313     }
1314
1315     if (masqueradesAsUndefinedWatchpointValid) {
1316         speculationWatchpointForMasqueradesAsUndefined();
1317         DFG_TYPE_CHECK(
1318             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1319                 MacroAssembler::Equal,
1320                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1321                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1322     } else {
1323         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1324         DFG_TYPE_CHECK(
1325             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1326                 MacroAssembler::Equal,
1327                 structureGPR,
1328                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1329         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1330             m_jit.branchTest8(
1331                 MacroAssembler::NonZero,
1332                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1333                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1334     }
1335
1336     // It seems that most of the time when programs do a == b where b may be either null/undefined
1337     // or an object, b is usually an object. Balance the branches to make that case fast.
1338     MacroAssembler::Jump rightNotCell =
1339         m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister);
1340
1341     // We know that within this branch, rightChild must be a cell.
1342     if (masqueradesAsUndefinedWatchpointValid) {
1343         speculationWatchpointForMasqueradesAsUndefined();
1344         DFG_TYPE_CHECK(
1345             JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr(
1346                 MacroAssembler::Equal,
1347                 MacroAssembler::Address(op2GPR, JSCell::structureOffset()),
1348                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1349     } else {
1350         m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR);
1351         DFG_TYPE_CHECK(
1352             JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr(
1353                 MacroAssembler::Equal,
1354                 structureGPR,
1355                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1356         speculationCheck(BadType, JSValueRegs(op2GPR), rightChild,
1357             m_jit.branchTest8(
1358                 MacroAssembler::NonZero,
1359                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1360                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1361     }
1362
1363     // At this point we know that we can perform a straight-forward equality comparison on pointer
1364     // values because both left and right are pointers to objects that have no special equality
1365     // protocols.
1366     MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR);
1367     MacroAssembler::Jump trueCase = m_jit.jump();
1368
1369     rightNotCell.link(&m_jit);
1370
1371     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1372     // prove that it is either null or undefined.
1373     if (needsTypeCheck(rightChild, SpecCell | SpecOther)) {
1374         m_jit.move(op2GPR, resultGPR);
1375         m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1376
1377         typeCheck(
1378             JSValueRegs(op2GPR), rightChild, SpecCell | SpecOther,
1379             m_jit.branch64(
1380                 MacroAssembler::NotEqual, resultGPR,
1381                 MacroAssembler::TrustedImm64(ValueNull)));
1382     }
1383
1384     falseCase.link(&m_jit);
1385     m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1386     MacroAssembler::Jump done = m_jit.jump();
1387     trueCase.link(&m_jit);
1388     m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1389     done.link(&m_jit);
1390
1391     jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1392 }
1393
1394 void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode)
1395 {
1396     BasicBlock* taken = branchNode->takenBlock();
1397     BasicBlock* notTaken = branchNode->notTakenBlock();
1398
1399     SpeculateCellOperand op1(this, leftChild);
1400     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1401     GPRTemporary result(this);
1402
1403     GPRReg op1GPR = op1.gpr();
1404     GPRReg op2GPR = op2.gpr();
1405     GPRReg resultGPR = result.gpr();
1406     GPRTemporary structure;
1407     GPRReg structureGPR = InvalidGPRReg;
1408
1409     bool masqueradesAsUndefinedWatchpointValid =
1410         masqueradesAsUndefinedWatchpointIsStillValid();
1411
1412     if (!masqueradesAsUndefinedWatchpointValid) {
1413         // The masquerades as undefined case will use the structure register, so allocate it here.
1414         // Do this at the top of the function to avoid branching around a register allocation.
1415         GPRTemporary realStructure(this);
1416         structure.adopt(realStructure);
1417         structureGPR = structure.gpr();
1418     }
1419
1420     if (masqueradesAsUndefinedWatchpointValid) {
1421         speculationWatchpointForMasqueradesAsUndefined();
1422         DFG_TYPE_CHECK(
1423             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1424                 MacroAssembler::Equal,
1425                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1426                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1427     } else {
1428         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1429         DFG_TYPE_CHECK(
1430             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1431                 MacroAssembler::Equal,
1432                 structureGPR,
1433                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1434         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1435             m_jit.branchTest8(
1436                 MacroAssembler::NonZero,
1437                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1438                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1439     }
1440
1441     // It seems that most of the time when programs do a == b where b may be either null/undefined
1442     // or an object, b is usually an object. Balance the branches to make that case fast.
1443     MacroAssembler::Jump rightNotCell =
1444         m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister);
1445
1446     // We know that within this branch, rightChild must be a cell.
1447     if (masqueradesAsUndefinedWatchpointValid) {
1448         speculationWatchpointForMasqueradesAsUndefined();
1449         DFG_TYPE_CHECK(
1450             JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr(
1451                 MacroAssembler::Equal,
1452                 MacroAssembler::Address(op2GPR, JSCell::structureOffset()),
1453                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1454     } else {
1455         m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR);
1456         DFG_TYPE_CHECK(
1457             JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr(
1458                 MacroAssembler::Equal,
1459                 structureGPR,
1460                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1461         speculationCheck(BadType, JSValueRegs(op2GPR), rightChild,
1462             m_jit.branchTest8(
1463                 MacroAssembler::NonZero,
1464                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1465                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1466     }
1467
1468     // At this point we know that we can perform a straight-forward equality comparison on pointer
1469     // values because both left and right are pointers to objects that have no special equality
1470     // protocols.
1471     branch64(MacroAssembler::Equal, op1GPR, op2GPR, taken);
1472
1473     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1474     // prove that it is either null or undefined.
1475     if (!needsTypeCheck(rightChild, SpecCell | SpecOther))
1476         rightNotCell.link(&m_jit);
1477     else {
1478         jump(notTaken, ForceJump);
1479
1480         rightNotCell.link(&m_jit);
1481         m_jit.move(op2GPR, resultGPR);
1482         m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1483
1484         typeCheck(
1485             JSValueRegs(op2GPR), rightChild, SpecCell | SpecOther, m_jit.branch64(
1486                 MacroAssembler::NotEqual, resultGPR,
1487                 MacroAssembler::TrustedImm64(ValueNull)));
1488     }
1489
1490     jump(notTaken);
1491 }
1492
1493 void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition)
1494 {
1495     SpeculateInt32Operand op1(this, node->child1());
1496     SpeculateInt32Operand op2(this, node->child2());
1497     GPRTemporary result(this, Reuse, op1, op2);
1498
1499     m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr());
1500
1501     // If we add a DataFormatBool, we should use it here.
1502     m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
1503     jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean);
1504 }
1505
1506 void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition)
1507 {
1508     SpeculateDoubleOperand op1(this, node->child1());
1509     SpeculateDoubleOperand op2(this, node->child2());
1510     GPRTemporary result(this);
1511
1512     m_jit.move(TrustedImm32(ValueTrue), result.gpr());
1513     MacroAssembler::Jump trueCase = m_jit.branchDouble(condition, op1.fpr(), op2.fpr());
1514     m_jit.xor64(TrustedImm32(true), result.gpr());
1515     trueCase.link(&m_jit);
1516
1517     jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1518 }
1519
1520 void SpeculativeJIT::compileValueAdd(Node* node)
1521 {
1522     JSValueOperand op1(this, node->child1());
1523     JSValueOperand op2(this, node->child2());
1524
1525     GPRReg op1GPR = op1.gpr();
1526     GPRReg op2GPR = op2.gpr();
1527
1528     flushRegisters();
1529
1530     GPRResult result(this);
1531     if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node()))
1532         callOperation(operationValueAddNotNumber, result.gpr(), op1GPR, op2GPR);
1533     else
1534         callOperation(operationValueAdd, result.gpr(), op1GPR, op2GPR);
1535
1536     jsValueResult(result.gpr(), node);
1537 }
1538
1539 void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse)
1540 {
1541     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1542     GPRTemporary result(this);
1543     GPRReg valueGPR = value.gpr();
1544     GPRReg resultGPR = result.gpr();
1545     GPRTemporary structure;
1546     GPRReg structureGPR = InvalidGPRReg;
1547
1548     bool masqueradesAsUndefinedWatchpointValid =
1549         masqueradesAsUndefinedWatchpointIsStillValid();
1550
1551     if (!masqueradesAsUndefinedWatchpointValid) {
1552         // The masquerades as undefined case will use the structure register, so allocate it here.
1553         // Do this at the top of the function to avoid branching around a register allocation.
1554         GPRTemporary realStructure(this);
1555         structure.adopt(realStructure);
1556         structureGPR = structure.gpr();
1557     }
1558
1559     MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister);
1560     if (masqueradesAsUndefinedWatchpointValid) {
1561         speculationWatchpointForMasqueradesAsUndefined();
1562         DFG_TYPE_CHECK(
1563             JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr(
1564                 MacroAssembler::Equal,
1565                 MacroAssembler::Address(valueGPR, JSCell::structureOffset()),
1566                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1567     } else {
1568         m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), structureGPR);
1569
1570         DFG_TYPE_CHECK(
1571             JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr(
1572                 MacroAssembler::Equal,
1573                 structureGPR,
1574                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1575
1576         MacroAssembler::Jump isNotMasqueradesAsUndefined =
1577             m_jit.branchTest8(
1578                 MacroAssembler::Zero,
1579                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1580                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined));
1581
1582         speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse,
1583             m_jit.branchPtr(
1584                 MacroAssembler::Equal,
1585                 MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()),
1586                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1587
1588         isNotMasqueradesAsUndefined.link(&m_jit);
1589     }
1590     m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1591     MacroAssembler::Jump done = m_jit.jump();
1592
1593     notCell.link(&m_jit);
1594
1595     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1596         m_jit.move(valueGPR, resultGPR);
1597         m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1598         typeCheck(
1599             JSValueRegs(valueGPR), nodeUse, SpecCell | SpecOther, m_jit.branch64(
1600                 MacroAssembler::NotEqual,
1601                 resultGPR,
1602                 MacroAssembler::TrustedImm64(ValueNull)));
1603     }
1604     m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1605
1606     done.link(&m_jit);
1607
1608     jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1609 }
1610
1611 void SpeculativeJIT::compileLogicalNot(Node* node)
1612 {
1613     switch (node->child1().useKind()) {
1614     case ObjectOrOtherUse: {
1615         compileObjectOrOtherLogicalNot(node->child1());
1616         return;
1617     }
1618
1619     case Int32Use: {
1620         SpeculateInt32Operand value(this, node->child1());
1621         GPRTemporary result(this, Reuse, value);
1622         m_jit.compare32(MacroAssembler::Equal, value.gpr(), MacroAssembler::TrustedImm32(0), result.gpr());
1623         m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
1624         jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1625         return;
1626     }
1627
1628     case NumberUse: {
1629         SpeculateDoubleOperand value(this, node->child1());
1630         FPRTemporary scratch(this);
1631         GPRTemporary result(this);
1632         m_jit.move(TrustedImm32(ValueFalse), result.gpr());
1633         MacroAssembler::Jump nonZero = m_jit.branchDoubleNonZero(value.fpr(), scratch.fpr());
1634         m_jit.xor32(TrustedImm32(true), result.gpr());
1635         nonZero.link(&m_jit);
1636         jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1637         return;
1638     }
1639
1640     case BooleanUse: {
1641         if (!needsTypeCheck(node->child1(), SpecBoolean)) {
1642             SpeculateBooleanOperand value(this, node->child1());
1643             GPRTemporary result(this, Reuse, value);
1644
1645             m_jit.move(value.gpr(), result.gpr());
1646             m_jit.xor64(TrustedImm32(true), result.gpr());
1647
1648             jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1649             return;
1650         }
1651
1652         JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
1653         GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add).
1654
1655         m_jit.move(value.gpr(), result.gpr());
1656         m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr());
1657         typeCheck(
1658             JSValueRegs(value.gpr()), node->child1(), SpecBoolean, m_jit.branchTest64(
1659                 JITCompiler::NonZero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1660         m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueTrue)), result.gpr());
1661
1662         // If we add a DataFormatBool, we should use it here.
1663         jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1664         return;
1665     }
1666
1667     case UntypedUse: {
1668         JSValueOperand arg1(this, node->child1());
1669         GPRTemporary result(this);
1670
1671         GPRReg arg1GPR = arg1.gpr();
1672         GPRReg resultGPR = result.gpr();
1673
1674         arg1.use();
1675
1676         m_jit.move(arg1GPR, resultGPR);
1677         m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), resultGPR);
1678         JITCompiler::Jump slowCase = m_jit.branchTest64(JITCompiler::NonZero, resultGPR, TrustedImm32(static_cast<int32_t>(~1)));
1679
1680         addSlowPathGenerator(
1681             slowPathCall(slowCase, this, dfgConvertJSValueToBoolean, resultGPR, arg1GPR));
1682
1683         m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueTrue)), resultGPR);
1684         jsValueResult(resultGPR, node, DataFormatJSBoolean, UseChildrenCalledExplicitly);
1685         return;
1686     }
1687
1688     default:
1689         RELEASE_ASSERT_NOT_REACHED();
1690         break;
1691     }
1692 }
1693
1694 void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, BasicBlock* notTaken)
1695 {
1696     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1697     GPRTemporary scratch(this);
1698     GPRReg valueGPR = value.gpr();
1699     GPRReg scratchGPR = scratch.gpr();
1700
1701     MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister);
1702     if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1703         speculationWatchpointForMasqueradesAsUndefined();
1704
1705         DFG_TYPE_CHECK(
1706             JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr(
1707                 MacroAssembler::Equal,
1708                 MacroAssembler::Address(valueGPR, JSCell::structureOffset()),
1709                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1710     } else {
1711         m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), scratchGPR);
1712
1713         DFG_TYPE_CHECK(
1714             JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr(
1715                 MacroAssembler::Equal,
1716                 scratchGPR,
1717                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1718
1719         JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined));
1720
1721         speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse,
1722             m_jit.branchPtr(
1723                 MacroAssembler::Equal,
1724                 MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()),
1725                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1726
1727         isNotMasqueradesAsUndefined.link(&m_jit);
1728     }
1729     jump(taken, ForceJump);
1730
1731     notCell.link(&m_jit);
1732
1733     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1734         m_jit.move(valueGPR, scratchGPR);
1735         m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), scratchGPR);
1736         typeCheck(
1737             JSValueRegs(valueGPR), nodeUse, SpecCell | SpecOther, m_jit.branch64(
1738                 MacroAssembler::NotEqual, scratchGPR, MacroAssembler::TrustedImm64(ValueNull)));
1739     }
1740     jump(notTaken);
1741
1742     noResult(m_currentNode);
1743 }
1744
1745 void SpeculativeJIT::emitBranch(Node* node)
1746 {
1747     BasicBlock* taken = node->takenBlock();
1748     BasicBlock* notTaken = node->notTakenBlock();
1749
1750     switch (node->child1().useKind()) {
1751     case ObjectOrOtherUse: {
1752         emitObjectOrOtherBranch(node->child1(), taken, notTaken);
1753         return;
1754     }
1755
1756     case Int32Use:
1757     case NumberUse: {
1758         if (node->child1().useKind() == Int32Use) {
1759             bool invert = false;
1760
1761             if (taken == nextBlock()) {
1762                 invert = true;
1763                 BasicBlock* tmp = taken;
1764                 taken = notTaken;
1765                 notTaken = tmp;
1766             }
1767
1768             SpeculateInt32Operand value(this, node->child1());
1769             branchTest32(invert ? MacroAssembler::Zero : MacroAssembler::NonZero, value.gpr(), taken);
1770         } else {
1771             SpeculateDoubleOperand value(this, node->child1());
1772             FPRTemporary scratch(this);
1773             branchDoubleNonZero(value.fpr(), scratch.fpr(), taken);
1774         }
1775
1776         jump(notTaken);
1777
1778         noResult(node);
1779         return;
1780     }
1781
1782     case UntypedUse:
1783     case BooleanUse: {
1784         JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
1785         GPRReg valueGPR = value.gpr();
1786
1787         if (node->child1().useKind() == BooleanUse) {
1788             if (!needsTypeCheck(node->child1(), SpecBoolean)) {
1789                 MacroAssembler::ResultCondition condition = MacroAssembler::NonZero;
1790
1791                 if (taken == nextBlock()) {
1792                     condition = MacroAssembler::Zero;
1793                     BasicBlock* tmp = taken;
1794                     taken = notTaken;
1795                     notTaken = tmp;
1796                 }
1797
1798                 branchTest32(condition, valueGPR, TrustedImm32(true), taken);
1799                 jump(notTaken);
1800             } else {
1801                 branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), notTaken);
1802                 branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), taken);
1803
1804                 typeCheck(JSValueRegs(valueGPR), node->child1(), SpecBoolean, m_jit.jump());
1805             }
1806             value.use();
1807         } else {
1808             GPRTemporary result(this);
1809             GPRReg resultGPR = result.gpr();
1810
1811             if (node->child1()->prediction() & SpecInt32) {
1812                 branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsNumber(0))), notTaken);
1813                 branch64(MacroAssembler::AboveOrEqual, valueGPR, GPRInfo::tagTypeNumberRegister, taken);
1814             }
1815
1816             if (node->child1()->prediction() & SpecBoolean) {
1817                 branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), notTaken);
1818                 branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), taken);
1819             }
1820
1821             value.use();
1822
1823             silentSpillAllRegisters(resultGPR);
1824             callOperation(dfgConvertJSValueToBoolean, resultGPR, valueGPR);
1825             silentFillAllRegisters(resultGPR);
1826
1827             branchTest32(MacroAssembler::NonZero, resultGPR, taken);
1828             jump(notTaken);
1829         }
1830
1831         noResult(node, UseChildrenCalledExplicitly);
1832         return;
1833     }
1834
1835     default:
1836         RELEASE_ASSERT_NOT_REACHED();
1837     }
1838 }
1839
1840 void SpeculativeJIT::compile(Node* node)
1841 {
1842     NodeType op = node->op();
1843
1844 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1845     m_jit.clearRegisterAllocationOffsets();
1846 #endif
1847
1848     switch (op) {
1849     case JSConstant:
1850         initConstantInfo(node);
1851         break;
1852
1853     case PhantomArguments:
1854         initConstantInfo(node);
1855         break;
1856
1857     case WeakJSConstant:
1858         m_jit.addWeakReference(node->weakConstant());
1859         initConstantInfo(node);
1860         break;
1861
1862     case Identity: {
1863         // CSE should always eliminate this.
1864         RELEASE_ASSERT_NOT_REACHED();
1865         break;
1866     }
1867
1868     case GetLocal: {
1869         SpeculatedType prediction = node->variableAccessData()->prediction();
1870         AbstractValue& value = m_state.variables().operand(node->local());
1871
1872         // If we have no prediction for this local, then don't attempt to compile.
1873         if (prediction == SpecNone) {
1874             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1875             break;
1876         }
1877
1878         // If the CFA is tracking this variable and it found that the variable
1879         // cannot have been assigned, then don't attempt to proceed.
1880         if (value.isClear()) {
1881             // FIXME: We should trap instead.
1882             // https://bugs.webkit.org/show_bug.cgi?id=110383
1883             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1884             break;
1885         }
1886
1887         switch (node->variableAccessData()->flushFormat()) {
1888         case FlushedDouble: {
1889             FPRTemporary result(this);
1890             m_jit.loadDouble(JITCompiler::addressFor(node->local()), result.fpr());
1891             VirtualRegister virtualRegister = node->virtualRegister();
1892             m_fprs.retain(result.fpr(), virtualRegister, SpillOrderDouble);
1893             generationInfoFromVirtualRegister(virtualRegister).initDouble(node, node->refCount(), result.fpr());
1894             break;
1895         }
1896
1897         case FlushedInt32: {
1898             GPRTemporary result(this);
1899             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
1900
1901             // Like int32Result, but don't useChildren - our children are phi nodes,
1902             // and don't represent values within this dataflow with virtual registers.
1903             VirtualRegister virtualRegister = node->virtualRegister();
1904             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderInteger);
1905             generationInfoFromVirtualRegister(virtualRegister).initInt32(node, node->refCount(), result.gpr());
1906             break;
1907         }
1908
1909         default:
1910             GPRTemporary result(this);
1911             m_jit.load64(JITCompiler::addressFor(node->local()), result.gpr());
1912
1913             // Like jsValueResult, but don't useChildren - our children are phi nodes,
1914             // and don't represent values within this dataflow with virtual registers.
1915             VirtualRegister virtualRegister = node->virtualRegister();
1916             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS);
1917
1918             DataFormat format;
1919             if (isCellSpeculation(value.m_type))
1920                 format = DataFormatJSCell;
1921             else if (isBooleanSpeculation(value.m_type))
1922                 format = DataFormatJSBoolean;
1923             else
1924                 format = DataFormatJS;
1925
1926             generationInfoFromVirtualRegister(virtualRegister).initJSValue(node, node->refCount(), result.gpr(), format);
1927             break;
1928         }
1929         break;
1930     }
1931
1932     case GetLocalUnlinked: {
1933         GPRTemporary result(this);
1934
1935         m_jit.load64(JITCompiler::addressFor(node->unlinkedLocal()), result.gpr());
1936
1937         jsValueResult(result.gpr(), node);
1938         break;
1939     }
1940
1941     case MovHintAndCheck: {
1942         compileMovHintAndCheck(node);
1943         break;
1944     }
1945
1946     case InlineStart: {
1947         compileInlineStart(node);
1948         break;
1949     }
1950
1951     case MovHint:
1952     case ZombieHint: {
1953         RELEASE_ASSERT_NOT_REACHED();
1954         break;
1955     }
1956
1957     case SetLocal: {
1958         // SetLocal doubles as a hint as to where a node will be stored and
1959         // as a speculation point. So before we speculate make sure that we
1960         // know where the child of this node needs to go in the virtual
1961         // stack.
1962         compileMovHint(node);
1963
1964         switch (node->variableAccessData()->flushFormat()) {
1965         case FlushedDouble: {
1966             SpeculateDoubleOperand value(this, node->child1());
1967             m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local()));
1968             noResult(node);
1969             // Indicate that it's no longer necessary to retrieve the value of
1970             // this bytecode variable from registers or other locations in the stack,
1971             // but that it is stored as a double.
1972             recordSetLocal(node->local(), ValueSource(DoubleInJSStack));
1973             break;
1974         }
1975
1976         case FlushedInt32: {
1977             SpeculateInt32Operand value(this, node->child1());
1978             m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local()));
1979             noResult(node);
1980             recordSetLocal(node->local(), ValueSource(Int32InJSStack));
1981             break;
1982         }
1983
1984         case FlushedCell: {
1985             SpeculateCellOperand cell(this, node->child1());
1986             GPRReg cellGPR = cell.gpr();
1987             m_jit.store64(cellGPR, JITCompiler::addressFor(node->local()));
1988             noResult(node);
1989             recordSetLocal(node->local(), ValueSource(CellInJSStack));
1990             break;
1991         }
1992
1993         case FlushedBoolean: {
1994             SpeculateBooleanOperand boolean(this, node->child1());
1995             m_jit.store64(boolean.gpr(), JITCompiler::addressFor(node->local()));
1996             noResult(node);
1997             recordSetLocal(node->local(), ValueSource(BooleanInJSStack));
1998             break;
1999         }
2000
2001         case FlushedJSValue: {
2002             JSValueOperand value(this, node->child1());
2003             m_jit.store64(value.gpr(), JITCompiler::addressFor(node->local()));
2004             noResult(node);
2005
2006             recordSetLocal(node->local(), ValueSource(ValueInJSStack));
2007
2008             // If we're storing an arguments object that has been optimized away,
2009             // our variable event stream for OSR exit now reflects the optimized
2010             // value (JSValue()). On the slow path, we want an arguments object
2011             // instead. We add an additional move hint to show OSR exit that it
2012             // needs to reconstruct the arguments object.
2013             if (node->child1()->op() == PhantomArguments)
2014                 compileMovHint(node);
2015             break;
2016         }
2017
2018         default:
2019             RELEASE_ASSERT_NOT_REACHED();
2020             break;
2021         }
2022
2023         break;
2024     }
2025
2026     case SetArgument:
2027         // This is a no-op; it just marks the fact that the argument is being used.
2028         // But it may be profitable to use this as a hook to run speculation checks
2029         // on arguments, thereby allowing us to trivially eliminate such checks if
2030         // the argument is not used.
2031         break;
2032
2033     case BitAnd:
2034     case BitOr:
2035     case BitXor:
2036         if (isInt32Constant(node->child1().node())) {
2037             SpeculateInt32Operand op2(this, node->child2());
2038             GPRTemporary result(this, Reuse, op2);
2039
2040             bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr());
2041
2042             int32Result(result.gpr(), node);
2043         } else if (isInt32Constant(node->child2().node())) {
2044             SpeculateInt32Operand op1(this, node->child1());
2045             GPRTemporary result(this, Reuse, op1);
2046
2047             bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr());
2048
2049             int32Result(result.gpr(), node);
2050         } else {
2051             SpeculateInt32Operand op1(this, node->child1());
2052             SpeculateInt32Operand op2(this, node->child2());
2053             GPRTemporary result(this, Reuse, op1, op2);
2054
2055             GPRReg reg1 = op1.gpr();
2056             GPRReg reg2 = op2.gpr();
2057             bitOp(op, reg1, reg2, result.gpr());
2058
2059             int32Result(result.gpr(), node);
2060         }
2061         break;
2062
2063     case BitRShift:
2064     case BitLShift:
2065     case BitURShift:
2066         if (isInt32Constant(node->child2().node())) {
2067             SpeculateInt32Operand op1(this, node->child1());
2068             GPRTemporary result(this, Reuse, op1);
2069
2070             shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr());
2071
2072             int32Result(result.gpr(), node);
2073         } else {
2074             // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
2075             SpeculateInt32Operand op1(this, node->child1());
2076             SpeculateInt32Operand op2(this, node->child2());
2077             GPRTemporary result(this, Reuse, op1);
2078
2079             GPRReg reg1 = op1.gpr();
2080             GPRReg reg2 = op2.gpr();
2081             shiftOp(op, reg1, reg2, result.gpr());
2082
2083             int32Result(result.gpr(), node);
2084         }
2085         break;
2086
2087     case UInt32ToNumber: {
2088         compileUInt32ToNumber(node);
2089         break;
2090     }
2091
2092     case DoubleAsInt32: {
2093         compileDoubleAsInt32(node);
2094         break;
2095     }
2096
2097     case ValueToInt32: {
2098         compileValueToInt32(node);
2099         break;
2100     }
2101
2102     case Int32ToDouble: {
2103         compileInt32ToDouble(node);
2104         break;
2105     }
2106
2107     case ValueAdd:
2108     case ArithAdd:
2109         compileAdd(node);
2110         break;
2111
2112     case MakeRope:
2113         compileMakeRope(node);
2114         break;
2115
2116     case ArithSub:
2117         compileArithSub(node);
2118         break;
2119
2120     case ArithNegate:
2121         compileArithNegate(node);
2122         break;
2123
2124     case ArithMul:
2125         compileArithMul(node);
2126         break;
2127
2128     case ArithIMul:
2129         compileArithIMul(node);
2130         break;
2131
2132     case ArithDiv: {
2133         compileArithDiv(node);
2134         break;
2135     }
2136
2137     case ArithMod: {
2138         compileArithMod(node);
2139         break;
2140     }
2141
2142     case ArithAbs: {
2143         switch (node->child1().useKind()) {
2144         case Int32Use: {
2145             SpeculateStrictInt32Operand op1(this, node->child1());
2146             GPRTemporary result(this);
2147             GPRTemporary scratch(this);
2148
2149             m_jit.move(op1.gpr(), result.gpr());
2150             m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr());
2151             m_jit.add32(scratch.gpr(), result.gpr());
2152             m_jit.xor32(scratch.gpr(), result.gpr());
2153             speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31)));
2154             int32Result(result.gpr(), node);
2155             break;
2156         }
2157
2158         case NumberUse: {
2159             SpeculateDoubleOperand op1(this, node->child1());
2160             FPRTemporary result(this);
2161
2162             m_jit.absDouble(op1.fpr(), result.fpr());
2163             doubleResult(result.fpr(), node);
2164             break;
2165         }
2166
2167         default:
2168             RELEASE_ASSERT_NOT_REACHED();
2169             break;
2170         }
2171         break;
2172     }
2173
2174     case ArithMin:
2175     case ArithMax: {
2176         switch (node->binaryUseKind()) {
2177         case Int32Use: {
2178             SpeculateStrictInt32Operand op1(this, node->child1());
2179             SpeculateStrictInt32Operand op2(this, node->child2());
2180             GPRTemporary result(this, Reuse, op1);
2181
2182             MacroAssembler::Jump op1Less = m_jit.branch32(op == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1.gpr(), op2.gpr());
2183             m_jit.move(op2.gpr(), result.gpr());
2184             if (op1.gpr() != result.gpr()) {
2185                 MacroAssembler::Jump done = m_jit.jump();
2186                 op1Less.link(&m_jit);
2187                 m_jit.move(op1.gpr(), result.gpr());
2188                 done.link(&m_jit);
2189             } else
2190                 op1Less.link(&m_jit);
2191
2192             int32Result(result.gpr(), node);
2193             break;
2194         }
2195
2196         case NumberUse: {
2197             SpeculateDoubleOperand op1(this, node->child1());
2198             SpeculateDoubleOperand op2(this, node->child2());
2199             FPRTemporary result(this, op1);
2200
2201             FPRReg op1FPR = op1.fpr();
2202             FPRReg op2FPR = op2.fpr();
2203             FPRReg resultFPR = result.fpr();
2204
2205             MacroAssembler::JumpList done;
2206
2207             MacroAssembler::Jump op1Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleLessThan : MacroAssembler::DoubleGreaterThan, op1FPR, op2FPR);
2208
2209             // op2 is eather the lesser one or one of then is NaN
2210             MacroAssembler::Jump op2Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleGreaterThanOrEqual : MacroAssembler::DoubleLessThanOrEqual, op1FPR, op2FPR);
2211
2212             // Unordered case. We don't know which of op1, op2 is NaN. Manufacture NaN by adding
2213             // op1 + op2 and putting it into result.
2214             m_jit.addDouble(op1FPR, op2FPR, resultFPR);
2215             done.append(m_jit.jump());
2216
2217             op2Less.link(&m_jit);
2218             m_jit.moveDouble(op2FPR, resultFPR);
2219
2220             if (op1FPR != resultFPR) {
2221                 done.append(m_jit.jump());
2222
2223                 op1Less.link(&m_jit);
2224                 m_jit.moveDouble(op1FPR, resultFPR);
2225             } else
2226                 op1Less.link(&m_jit);
2227
2228             done.link(&m_jit);
2229
2230             doubleResult(resultFPR, node);
2231             break;
2232         }
2233
2234         default:
2235             RELEASE_ASSERT_NOT_REACHED();
2236             break;
2237         }
2238         break;
2239     }
2240
2241     case ArithSqrt: {
2242         SpeculateDoubleOperand op1(this, node->child1());
2243         FPRTemporary result(this, op1);
2244
2245         m_jit.sqrtDouble(op1.fpr(), result.fpr());
2246
2247         doubleResult(result.fpr(), node);
2248         break;
2249     }
2250
2251     case LogicalNot:
2252         compileLogicalNot(node);
2253         break;
2254
2255     case CompareLess:
2256         if (compare(node, JITCompiler::LessThan, JITCompiler::DoubleLessThan, operationCompareLess))
2257             return;
2258         break;
2259
2260     case CompareLessEq:
2261         if (compare(node, JITCompiler::LessThanOrEqual, JITCompiler::DoubleLessThanOrEqual, operationCompareLessEq))
2262             return;
2263         break;
2264
2265     case CompareGreater:
2266         if (compare(node, JITCompiler::GreaterThan, JITCompiler::DoubleGreaterThan, operationCompareGreater))
2267             return;
2268         break;
2269
2270     case CompareGreaterEq:
2271         if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq))
2272             return;
2273         break;
2274
2275     case CompareEqConstant:
2276         ASSERT(isNullConstant(node->child2().node()));
2277         if (nonSpeculativeCompareNull(node, node->child1()))
2278             return;
2279         break;
2280
2281     case CompareEq:
2282         if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq))
2283             return;
2284         break;
2285
2286     case CompareStrictEqConstant:
2287         if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node())))
2288             return;
2289         break;
2290
2291     case CompareStrictEq:
2292         if (compileStrictEq(node))
2293             return;
2294         break;
2295
2296     case StringCharCodeAt: {
2297         compileGetCharCodeAt(node);
2298         break;
2299     }
2300
2301     case StringCharAt: {
2302         // Relies on StringCharAt node having same basic layout as GetByVal
2303         compileGetByValOnString(node);
2304         break;
2305     }
2306
2307     case StringFromCharCode: {
2308         compileFromCharCode(node);
2309         break;
2310     }
2311
2312     case CheckArray: {
2313         checkArray(node);
2314         break;
2315     }
2316
2317     case Arrayify:
2318     case ArrayifyToStructure: {
2319         arrayify(node);
2320         break;
2321     }
2322
2323     case GetByVal: {
2324         switch (node->arrayMode().type()) {
2325         case Array::SelectUsingPredictions:
2326         case Array::ForceExit:
2327             RELEASE_ASSERT_NOT_REACHED();
2328             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2329             break;
2330         case Array::Generic: {
2331             JSValueOperand base(this, node->child1());
2332             JSValueOperand property(this, node->child2());
2333             GPRReg baseGPR = base.gpr();
2334             GPRReg propertyGPR = property.gpr();
2335
2336             flushRegisters();
2337             GPRResult result(this);
2338             callOperation(operationGetByVal, result.gpr(), baseGPR, propertyGPR);
2339
2340             jsValueResult(result.gpr(), node);
2341             break;
2342         }
2343         case Array::Int32:
2344         case Array::Contiguous: {
2345             if (node->arrayMode().isInBounds()) {
2346                 SpeculateStrictInt32Operand property(this, node->child2());
2347                 StorageOperand storage(this, node->child3());
2348
2349                 GPRReg propertyReg = property.gpr();
2350                 GPRReg storageReg = storage.gpr();
2351
2352                 if (!m_compileOkay)
2353                     return;
2354
2355                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2356
2357                 GPRTemporary result(this);
2358                 m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.gpr());
2359                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr()));
2360                 jsValueResult(result.gpr(), node, node->arrayMode().type() == Array::Int32 ? DataFormatJSInt32 : DataFormatJS);
2361                 break;
2362             }
2363
2364             SpeculateCellOperand base(this, node->child1());
2365             SpeculateStrictInt32Operand property(this, node->child2());
2366             StorageOperand storage(this, node->child3());
2367
2368             GPRReg baseReg = base.gpr();
2369             GPRReg propertyReg = property.gpr();
2370             GPRReg storageReg = storage.gpr();
2371
2372             if (!m_compileOkay)
2373                 return;
2374
2375             GPRTemporary result(this);
2376             GPRReg resultReg = result.gpr();
2377
2378             MacroAssembler::JumpList slowCases;
2379
2380             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2381
2382             m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
2383             slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, resultReg));
2384
2385             addSlowPathGenerator(
2386                 slowPathCall(
2387                     slowCases, this, operationGetByValArrayInt,
2388                     result.gpr(), baseReg, propertyReg));
2389
2390             jsValueResult(resultReg, node);
2391             break;
2392         }
2393
2394         case Array::Double: {
2395             if (node->arrayMode().isInBounds()) {
2396                 if (node->arrayMode().isSaneChain()) {
2397                     JSGlobalObject* globalObject = m_jit.globalObjectFor(node->codeOrigin);
2398                     ASSERT(globalObject->arrayPrototypeChainIsSane());
2399                     m_jit.addLazily(
2400                         speculationWatchpoint(),
2401                         globalObject->arrayPrototype()->structure()->transitionWatchpointSet());
2402                     m_jit.addLazily(
2403                         speculationWatchpoint(),
2404                         globalObject->objectPrototype()->structure()->transitionWatchpointSet());
2405                 }
2406
2407                 SpeculateStrictInt32Operand property(this, node->child2());
2408                 StorageOperand storage(this, node->child3());
2409
2410                 GPRReg propertyReg = property.gpr();
2411                 GPRReg storageReg = storage.gpr();
2412
2413                 if (!m_compileOkay)
2414                     return;
2415
2416                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2417
2418                 FPRTemporary result(this);
2419                 m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.fpr());
2420                 if (!node->arrayMode().isSaneChain())
2421                     speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, result.fpr(), result.fpr()));
2422                 doubleResult(result.fpr(), node);
2423                 break;
2424             }
2425
2426             SpeculateCellOperand base(this, node->child1());
2427             SpeculateStrictInt32Operand property(this, node->child2());
2428             StorageOperand storage(this, node->child3());
2429
2430             GPRReg baseReg = base.gpr();
2431             GPRReg propertyReg = property.gpr();
2432             GPRReg storageReg = storage.gpr();
2433
2434             if (!m_compileOkay)
2435                 return;
2436
2437             GPRTemporary result(this);
2438             FPRTemporary temp(this);
2439             GPRReg resultReg = result.gpr();
2440             FPRReg tempReg = temp.fpr();
2441
2442             MacroAssembler::JumpList slowCases;
2443
2444             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2445
2446             m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), tempReg);
2447             slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempReg, tempReg));
2448             boxDouble(tempReg, resultReg);
2449
2450             addSlowPathGenerator(
2451                 slowPathCall(
2452                     slowCases, this, operationGetByValArrayInt,
2453                     result.gpr(), baseReg, propertyReg));
2454
2455             jsValueResult(resultReg, node);
2456             break;
2457         }
2458
2459         case Array::ArrayStorage:
2460         case Array::SlowPutArrayStorage: {
2461             if (node->arrayMode().isInBounds()) {
2462                 SpeculateStrictInt32Operand property(this, node->child2());
2463                 StorageOperand storage(this, node->child3());
2464
2465                 GPRReg propertyReg = property.gpr();
2466                 GPRReg storageReg = storage.gpr();
2467
2468                 if (!m_compileOkay)
2469                     return;
2470
2471                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2472
2473                 GPRTemporary result(this);
2474                 m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), result.gpr());
2475                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr()));
2476
2477                 jsValueResult(result.gpr(), node);
2478                 break;
2479             }
2480
2481             SpeculateCellOperand base(this, node->child1());
2482             SpeculateStrictInt32Operand property(this, node->child2());
2483             StorageOperand storage(this, node->child3());
2484
2485             GPRReg baseReg = base.gpr();
2486             GPRReg propertyReg = property.gpr();
2487             GPRReg storageReg = storage.gpr();
2488
2489             if (!m_compileOkay)
2490                 return;
2491
2492             GPRTemporary result(this);
2493             GPRReg resultReg = result.gpr();
2494
2495             MacroAssembler::JumpList slowCases;
2496
2497             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2498
2499             m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), resultReg);
2500             slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, resultReg));
2501
2502             addSlowPathGenerator(
2503                 slowPathCall(
2504                     slowCases, this, operationGetByValArrayInt,
2505                     result.gpr(), baseReg, propertyReg));
2506
2507             jsValueResult(resultReg, node);
2508             break;
2509         }
2510         case Array::String:
2511             compileGetByValOnString(node);
2512             break;
2513         case Array::Arguments:
2514             compileGetByValOnArguments(node);
2515             break;
2516         default: {
2517             TypedArrayType type = node->arrayMode().typedArrayType();
2518             if (isInt(type))
2519                 compileGetByValOnIntTypedArray(node, type);
2520             else
2521                 compileGetByValOnFloatTypedArray(node, type);
2522         } }
2523         break;
2524     }
2525
2526     case PutByVal:
2527     case PutByValAlias: {
2528         Edge child1 = m_jit.graph().varArgChild(node, 0);
2529         Edge child2 = m_jit.graph().varArgChild(node, 1);
2530         Edge child3 = m_jit.graph().varArgChild(node, 2);
2531         Edge child4 = m_jit.graph().varArgChild(node, 3);
2532
2533         ArrayMode arrayMode = node->arrayMode().modeForPut();
2534         bool alreadyHandled = false;
2535
2536         switch (arrayMode.type()) {
2537         case Array::SelectUsingPredictions:
2538         case Array::ForceExit:
2539             RELEASE_ASSERT_NOT_REACHED();
2540             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2541             alreadyHandled = true;
2542             break;
2543         case Array::Generic: {
2544             RELEASE_ASSERT(node->op() == PutByVal);
2545
2546             JSValueOperand arg1(this, child1);
2547             JSValueOperand arg2(this, child2);
2548             JSValueOperand arg3(this, child3);
2549             GPRReg arg1GPR = arg1.gpr();
2550             GPRReg arg2GPR = arg2.gpr();
2551             GPRReg arg3GPR = arg3.gpr();
2552             flushRegisters();
2553
2554             callOperation(m_jit.strictModeFor(node->codeOrigin) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR);
2555
2556             noResult(node);
2557             alreadyHandled = true;
2558             break;
2559         }
2560         default:
2561             break;
2562         }
2563
2564         if (alreadyHandled)
2565             break;
2566
2567         // FIXME: the base may not be necessary for some array access modes. But we have to
2568         // keep it alive to this point, so it's likely to be in a register anyway. Likely
2569         // no harm in locking it here.
2570         SpeculateCellOperand base(this, child1);
2571         SpeculateStrictInt32Operand property(this, child2);
2572
2573         GPRReg baseReg = base.gpr();
2574         GPRReg propertyReg = property.gpr();
2575
2576         switch (arrayMode.type()) {
2577         case Array::Int32:
2578         case Array::Contiguous: {
2579             JSValueOperand value(this, child3, ManualOperandSpeculation);
2580
2581             GPRReg valueReg = value.gpr();
2582
2583             if (!m_compileOkay)
2584                 return;
2585
2586             if (arrayMode.type() == Array::Int32) {
2587                 DFG_TYPE_CHECK(
2588                     JSValueRegs(valueReg), child3, SpecInt32,
2589                     m_jit.branch64(
2590                         MacroAssembler::Below, valueReg, GPRInfo::tagTypeNumberRegister));
2591             }
2592
2593             if (arrayMode.type() == Array::Contiguous && Heap::isWriteBarrierEnabled()) {
2594                 GPRTemporary scratch(this);
2595                 writeBarrier(baseReg, value.gpr(), child3, WriteBarrierForPropertyAccess, scratch.gpr());
2596             }
2597
2598             StorageOperand storage(this, child4);
2599             GPRReg storageReg = storage.gpr();
2600
2601             if (node->op() == PutByValAlias) {
2602                 // Store the value to the array.
2603                 GPRReg propertyReg = property.gpr();
2604                 GPRReg valueReg = value.gpr();
2605                 m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2606
2607                 noResult(node);
2608                 break;
2609             }
2610
2611             GPRTemporary temporary;
2612             GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2613
2614             MacroAssembler::Jump slowCase;
2615
2616             if (arrayMode.isInBounds()) {
2617                 speculationCheck(
2618                     StoreToHoleOrOutOfBounds, JSValueRegs(), 0,
2619                     m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2620             } else {
2621                 MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2622
2623                 slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2624
2625                 if (!arrayMode.isOutOfBounds())
2626                     speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2627
2628                 m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2629                 m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2630
2631                 inBounds.link(&m_jit);
2632             }
2633
2634             m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2635
2636             base.use();
2637             property.use();
2638             value.use();
2639             storage.use();
2640
2641             if (arrayMode.isOutOfBounds()) {
2642                 addSlowPathGenerator(
2643                     slowPathCall(
2644                         slowCase, this,
2645                         m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2646                         NoResult, baseReg, propertyReg, valueReg));
2647             }
2648
2649             noResult(node, UseChildrenCalledExplicitly);
2650             break;
2651         }
2652
2653         case Array::Double: {
2654             compileDoublePutByVal(node, base, property);
2655             break;
2656         }
2657
2658         case Array::ArrayStorage:
2659         case Array::SlowPutArrayStorage: {
2660             JSValueOperand value(this, child3);
2661
2662             GPRReg valueReg = value.gpr();
2663
2664             if (!m_compileOkay)
2665                 return;
2666
2667             if (Heap::isWriteBarrierEnabled()) {
2668                 GPRTemporary scratch(this);
2669                 writeBarrier(baseReg, value.gpr(), child3, WriteBarrierForPropertyAccess, scratch.gpr());
2670             }
2671
2672             StorageOperand storage(this, child4);
2673             GPRReg storageReg = storage.gpr();
2674
2675             if (node->op() == PutByValAlias) {
2676                 // Store the value to the array.
2677                 GPRReg propertyReg = property.gpr();
2678                 GPRReg valueReg = value.gpr();
2679                 m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2680
2681                 noResult(node);
2682                 break;
2683             }
2684
2685             GPRTemporary temporary;
2686             GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2687
2688             MacroAssembler::JumpList slowCases;
2689
2690             MacroAssembler::Jump beyondArrayBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2691             if (!arrayMode.isOutOfBounds())
2692                 speculationCheck(OutOfBounds, JSValueRegs(), 0, beyondArrayBounds);
2693             else
2694                 slowCases.append(beyondArrayBounds);
2695
2696             // Check if we're writing to a hole; if so increment m_numValuesInVector.
2697             if (arrayMode.isInBounds()) {
2698                 speculationCheck(
2699                     StoreToHole, JSValueRegs(), 0,
2700                     m_jit.branchTest64(MacroAssembler::Zero, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]))));
2701             } else {
2702                 MacroAssembler::Jump notHoleValue = m_jit.branchTest64(MacroAssembler::NonZero, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2703                 if (arrayMode.isSlowPut()) {
2704                     // This is sort of strange. If we wanted to optimize this code path, we would invert
2705                     // the above branch. But it's simply not worth it since this only happens if we're
2706                     // already having a bad time.
2707                     slowCases.append(m_jit.jump());
2708                 } else {
2709                     m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageReg, ArrayStorage::numValuesInVectorOffset()));
2710
2711                     // If we're writing to a hole we might be growing the array;
2712                     MacroAssembler::Jump lengthDoesNotNeedUpdate = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2713                     m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2714                     m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2715
2716                     lengthDoesNotNeedUpdate.link(&m_jit);
2717                 }
2718                 notHoleValue.link(&m_jit);
2719             }
2720
2721             // Store the value to the array.
2722             m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2723
2724             base.use();
2725             property.use();
2726             value.use();
2727             storage.use();
2728
2729             if (!slowCases.empty()) {
2730                 addSlowPathGenerator(
2731                     slowPathCall(
2732                         slowCases, this,
2733                         m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2734                         NoResult, baseReg, propertyReg, valueReg));
2735             }
2736
2737             noResult(node, UseChildrenCalledExplicitly);
2738             break;
2739         }
2740
2741         case Array::Arguments: {
2742             JSValueOperand value(this, child3);
2743             GPRTemporary scratch(this);
2744             GPRTemporary scratch2(this);
2745
2746             GPRReg valueReg = value.gpr();
2747             GPRReg scratchReg = scratch.gpr();
2748             GPRReg scratch2Reg = scratch2.gpr();
2749
2750             if (!m_compileOkay)
2751                 return;
2752
2753             // Two really lame checks.
2754             speculationCheck(
2755                 Uncountable, JSValueSource(), 0,
2756                 m_jit.branch32(
2757                     MacroAssembler::AboveOrEqual, propertyReg,
2758                     MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_numArguments))));
2759             speculationCheck(
2760                 Uncountable, JSValueSource(), 0,
2761                 m_jit.branchTestPtr(
2762                     MacroAssembler::NonZero,
2763                     MacroAssembler::Address(
2764                         baseReg, OBJECT_OFFSETOF(Arguments, m_slowArguments))));
2765
2766             m_jit.move(propertyReg, scratch2Reg);
2767             m_jit.signExtend32ToPtr(scratch2Reg, scratch2Reg);
2768             m_jit.loadPtr(
2769                 MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_registers)),
2770                 scratchReg);
2771
2772             m_jit.store64(
2773                 valueReg,
2774                 MacroAssembler::BaseIndex(
2775                     scratchReg, scratch2Reg, MacroAssembler::TimesEight,
2776                     CallFrame::thisArgumentOffset() * sizeof(Register) + sizeof(Register)));
2777
2778             noResult(node);
2779             break;
2780         }
2781
2782         default: {
2783             TypedArrayType type = arrayMode.typedArrayType();
2784             if (isInt(type))
2785                 compilePutByValForIntTypedArray(base.gpr(), property.gpr(), node, type);
2786             else
2787                 compilePutByValForFloatTypedArray(base.gpr(), property.gpr(), node, type);
2788         } }
2789
2790         break;
2791     }
2792
2793     case RegExpExec: {
2794         if (compileRegExpExec(node))
2795             return;
2796         if (!node->adjustedRefCount()) {
2797             SpeculateCellOperand base(this, node->child1());
2798             SpeculateCellOperand argument(this, node->child2());
2799             GPRReg baseGPR = base.gpr();
2800             GPRReg argumentGPR = argument.gpr();
2801
2802             flushRegisters();
2803             GPRResult result(this);
2804             callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2805
2806             // Must use jsValueResult because otherwise we screw up register
2807             // allocation, which thinks that this node has a result.
2808             jsValueResult(result.gpr(), node);
2809             break;
2810         }
2811
2812         SpeculateCellOperand base(this, node->child1());
2813         SpeculateCellOperand argument(this, node->child2());
2814         GPRReg baseGPR = base.gpr();
2815         GPRReg argumentGPR = argument.gpr();
2816
2817         flushRegisters();
2818         GPRResult result(this);
2819         callOperation(operationRegExpExec, result.gpr(), baseGPR, argumentGPR);
2820
2821         jsValueResult(result.gpr(), node);
2822         break;
2823     }
2824
2825     case RegExpTest: {
2826         SpeculateCellOperand base(this, node->child1());
2827         SpeculateCellOperand argument(this, node->child2());
2828         GPRReg baseGPR = base.gpr();
2829         GPRReg argumentGPR = argument.gpr();
2830
2831         flushRegisters();
2832         GPRResult result(this);
2833         callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2834
2835         // If we add a DataFormatBool, we should use it here.
2836         m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
2837         jsValueResult(result.gpr(), node, DataFormatJSBoolean);
2838         break;
2839     }
2840
2841     case ArrayPush: {
2842         ASSERT(node->arrayMode().isJSArray());
2843
2844         SpeculateCellOperand base(this, node->child1());
2845         GPRTemporary storageLength(this);
2846
2847         GPRReg baseGPR = base.gpr();
2848         GPRReg storageLengthGPR = storageLength.gpr();
2849
2850         StorageOperand storage(this, node->child3());
2851         GPRReg storageGPR = storage.gpr();
2852
2853         switch (node->arrayMode().type()) {
2854         case Array::Int32:
2855         case Array::Contiguous: {
2856             JSValueOperand value(this, node->child2(), ManualOperandSpeculation);
2857             GPRReg valueGPR = value.gpr();
2858
2859             if (node->arrayMode().type() == Array::Int32) {
2860                 DFG_TYPE_CHECK(
2861                     JSValueRegs(valueGPR), node->child2(), SpecInt32,
2862                     m_jit.branch64(
2863                         MacroAssembler::Below, valueGPR, GPRInfo::tagTypeNumberRegister));
2864             }
2865
2866             if (node->arrayMode().type() != Array::Int32 && Heap::isWriteBarrierEnabled()) {
2867                 GPRTemporary scratch(this);
2868                 writeBarrier(baseGPR, valueGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
2869             }
2870
2871             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2872             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
2873             m_jit.store64(valueGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2874             m_jit.add32(TrustedImm32(1), storageLengthGPR);
2875             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2876             m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2877
2878             addSlowPathGenerator(
2879                 slowPathCall(
2880                     slowPath, this, operationArrayPush, NoResult, storageLengthGPR,
2881                     valueGPR, baseGPR));
2882
2883             jsValueResult(storageLengthGPR, node);
2884             break;
2885         }
2886
2887         case Array::Double: {
2888             SpeculateDoubleOperand value(this, node->child2());
2889             FPRReg valueFPR = value.fpr();
2890
2891             DFG_TYPE_CHECK(
2892                 JSValueRegs(), node->child2(), SpecRealNumber,
2893                 m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR));
2894
2895             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2896             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
2897             m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2898             m_jit.add32(TrustedImm32(1), storageLengthGPR);
2899             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2900             m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2901
2902             addSlowPathGenerator(
2903                 slowPathCall(
2904                     slowPath, this, operationArrayPushDouble, NoResult, storageLengthGPR,
2905                     valueFPR, baseGPR));
2906
2907             jsValueResult(storageLengthGPR, node);
2908             break;
2909         }
2910
2911         case Array::ArrayStorage: {
2912             JSValueOperand value(this, node->child2());
2913             GPRReg valueGPR = value.gpr();
2914
2915             if (Heap::isWriteBarrierEnabled()) {
2916                 GPRTemporary scratch(this);
2917                 writeBarrier(baseGPR, valueGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
2918             }
2919
2920             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
2921
2922             // Refuse to handle bizarre lengths.
2923             speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(0x7ffffffe)));
2924
2925             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
2926
2927             m_jit.store64(valueGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2928
2929             m_jit.add32(TrustedImm32(1), storageLengthGPR);
2930             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
2931             m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
2932             m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2933
2934             addSlowPathGenerator(
2935                 slowPathCall(
2936                     slowPath, this, operationArrayPush, NoResult, storageLengthGPR,
2937                     valueGPR, baseGPR));
2938
2939             jsValueResult(storageLengthGPR, node);
2940             break;
2941         }
2942
2943         default:
2944             CRASH();
2945             break;
2946         }
2947         break;
2948     }
2949
2950     case ArrayPop: {
2951         ASSERT(node->arrayMode().isJSArray());
2952
2953         SpeculateCellOperand base(this, node->child1());
2954         StorageOperand storage(this, node->child2());
2955         GPRTemporary value(this);
2956         GPRTemporary storageLength(this);
2957         FPRTemporary temp(this); // This is kind of lame, since we don't always need it. I'm relying on the fact that we don't have FPR pressure, especially in code that uses pop().
2958
2959         GPRReg baseGPR = base.gpr();
2960         GPRReg storageGPR = storage.gpr();
2961         GPRReg valueGPR = value.gpr();
2962         GPRReg storageLengthGPR = storageLength.gpr();
2963         FPRReg tempFPR = temp.fpr();
2964
2965         switch (node->arrayMode().type()) {
2966         case Array::Int32:
2967         case Array::Double:
2968         case Array::Contiguous: {
2969             m_jit.load32(
2970                 MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2971             MacroAssembler::Jump undefinedCase =
2972                 m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR);
2973             m_jit.sub32(TrustedImm32(1), storageLengthGPR);
2974             m_jit.store32(
2975                 storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2976             MacroAssembler::Jump slowCase;
2977             if (node->arrayMode().type() == Array::Double) {
2978                 m_jit.loadDouble(
2979                     MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight),
2980                     tempFPR);
2981                 // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old
2982                 // length and the new length.
2983                 m_jit.store64(
2984                 MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2985                 slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR);
2986                 boxDouble(tempFPR, valueGPR);
2987             } else {
2988                 m_jit.load64(
2989                     MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight),
2990                     valueGPR);
2991                 // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old
2992                 // length and the new length.
2993                 m_jit.store64(
2994                 MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2995                 slowCase = m_jit.branchTest64(MacroAssembler::Zero, valueGPR);
2996             }
2997
2998             addSlowPathGenerator(
2999                 slowPathMove(
3000                     undefinedCase, this,
3001                     MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), valueGPR));
3002             addSlowPathGenerator(
3003                 slowPathCall(
3004                     slowCase, this, operationArrayPopAndRecoverLength, valueGPR, baseGPR));
3005
3006             // We can't know for sure that the result is an int because of the slow paths. :-/
3007             jsValueResult(valueGPR, node);
3008             break;
3009         }
3010
3011         case Array::ArrayStorage: {
3012             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3013
3014             JITCompiler::Jump undefinedCase =
3015                 m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR);
3016
3017             m_jit.sub32(TrustedImm32(1), storageLengthGPR);
3018
3019             JITCompiler::JumpList slowCases;
3020             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset())));
3021
3022             m_jit.load64(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), valueGPR);
3023             slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, valueGPR));
3024
3025             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3026
3027             m_jit.store64(MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
3028             m_jit.sub32(MacroAssembler::TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3029
3030             addSlowPathGenerator(
3031                 slowPathMove(
3032                     undefinedCase, this,
3033                     MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), valueGPR));
3034
3035             addSlowPathGenerator(
3036                 slowPathCall(
3037                     slowCases, this, operationArrayPop, valueGPR, baseGPR));
3038
3039             jsValueResult(valueGPR, node);
3040             break;
3041         }
3042
3043         default:
3044             CRASH();
3045             break;
3046         }
3047         break;
3048     }
3049
3050     case DFG::Jump: {
3051         jump(node->takenBlock());
3052         noResult(node);
3053         break;
3054     }
3055
3056     case Branch:
3057         emitBranch(node);
3058         break;
3059
3060     case Switch:
3061         emitSwitch(node);
3062         break;
3063
3064     case Return: {
3065         ASSERT(GPRInfo::callFrameRegister != GPRInfo::regT1);
3066         ASSERT(GPRInfo::regT1 != GPRInfo::returnValueGPR);
3067         ASSERT(GPRInfo::returnValueGPR != GPRInfo::callFrameRegister);
3068
3069 #if DFG_ENABLE(SUCCESS_STATS)
3070         static SamplingCounter counter("SpeculativeJIT");
3071         m_jit.emitCount(counter);
3072 #endif
3073
3074         // Return the result in returnValueGPR.
3075         JSValueOperand op1(this, node->child1());
3076         m_jit.move(op1.gpr(), GPRInfo::returnValueGPR);
3077
3078         // Grab the return address.
3079         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, GPRInfo::regT1);
3080         // Restore our caller's "r".
3081         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, GPRInfo::callFrameRegister);
3082         // Return.
3083         m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT1);
3084         m_jit.ret();
3085
3086         noResult(node);
3087         break;
3088     }
3089
3090     case Throw:
3091     case ThrowReferenceError: {
3092         // We expect that throw statements are rare and are intended to exit the code block
3093         // anyway, so we just OSR back to the old JIT for now.
3094         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
3095         break;
3096     }
3097
3098     case ToPrimitive: {
3099         RELEASE_ASSERT(node->child1().useKind() == UntypedUse);
3100         JSValueOperand op1(this, node->child1());
3101         GPRTemporary result(this, Reuse, op1);
3102
3103         GPRReg op1GPR = op1.gpr();
3104         GPRReg resultGPR = result.gpr();
3105
3106         op1.use();
3107
3108         if (!(m_state.forNode(node->child1()).m_type & ~(SpecNumber | SpecBoolean)))
3109             m_jit.move(op1GPR, resultGPR);
3110         else {
3111             MacroAssembler::Jump alreadyPrimitive = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagMaskRegister);
3112             MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1GPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3113
3114             alreadyPrimitive.link(&m_jit);
3115             m_jit.move(op1GPR, resultGPR);
3116
3117             addSlowPathGenerator(
3118                 slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, op1GPR));
3119         }
3120
3121         jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3122         break;
3123     }
3124
3125     case ToString: {
3126         if (node->child1().useKind() == UntypedUse) {
3127             JSValueOperand op1(this, node->child1());
3128             GPRReg op1GPR = op1.gpr();
3129
3130             GPRResult result(this);
3131             GPRReg resultGPR = result.gpr();
3132
3133             flushRegisters();
3134
3135             JITCompiler::Jump done;
3136             if (node->child1()->prediction() & SpecString) {
3137                 JITCompiler::Jump slowPath1 = m_jit.branchTest64(
3138                     JITCompiler::NonZero, op1GPR, GPRInfo::tagMaskRegister);
3139                 JITCompiler::Jump slowPath2 = m_jit.branchPtr(
3140                     JITCompiler::NotEqual,
3141                     JITCompiler::Address(op1GPR, JSCell::structureOffset()),
3142                     TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3143                 m_jit.move(op1GPR, resultGPR);
3144                 done = m_jit.jump();
3145                 slowPath1.link(&m_jit);
3146                 slowPath2.link(&m_jit);
3147             }
3148             callOperation(operationToString, resultGPR, op1GPR);
3149             if (done.isSet())
3150                 done.link(&m_jit);
3151             cellResult(resultGPR, node);
3152             break;
3153         }
3154
3155         compileToStringOnCell(node);
3156         break;
3157     }
3158
3159     case NewStringObject: {
3160         compileNewStringObject(node);
3161         break;
3162     }
3163
3164     case NewArray: {