abe65185526f243fa71f3c23a6ea0de4bd7e8e7a
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToB3.cpp
1 /*
2  * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToB3.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "AirCode.h"
32 #include "AirGenerationContext.h"
33 #include "AllowMacroScratchRegisterUsage.h"
34 #include "AllowMacroScratchRegisterUsageIf.h"
35 #include "AtomicsObject.h"
36 #include "B3CheckValue.h"
37 #include "B3FenceValue.h"
38 #include "B3PatchpointValue.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3ValueInlines.h"
42 #include "CallFrameShuffler.h"
43 #include "CodeBlockWithJITType.h"
44 #include "DFGAbstractInterpreterInlines.h"
45 #include "DFGCapabilities.h"
46 #include "DFGDoesGC.h"
47 #include "DFGDominators.h"
48 #include "DFGInPlaceAbstractState.h"
49 #include "DFGLivenessAnalysisPhase.h"
50 #include "DFGMayExit.h"
51 #include "DFGOSRAvailabilityAnalysisPhase.h"
52 #include "DFGOSRExitFuzz.h"
53 #include "DirectArguments.h"
54 #include "FTLAbstractHeapRepository.h"
55 #include "FTLAvailableRecovery.h"
56 #include "FTLExceptionTarget.h"
57 #include "FTLForOSREntryJITCode.h"
58 #include "FTLFormattedValue.h"
59 #include "FTLLazySlowPathCall.h"
60 #include "FTLLoweredNodeValue.h"
61 #include "FTLOperations.h"
62 #include "FTLOutput.h"
63 #include "FTLPatchpointExceptionHandle.h"
64 #include "FTLSnippetParams.h"
65 #include "FTLThunks.h"
66 #include "FTLWeightedTarget.h"
67 #include "JITAddGenerator.h"
68 #include "JITBitAndGenerator.h"
69 #include "JITBitOrGenerator.h"
70 #include "JITBitXorGenerator.h"
71 #include "JITDivGenerator.h"
72 #include "JITInlineCacheGenerator.h"
73 #include "JITLeftShiftGenerator.h"
74 #include "JITMathIC.h"
75 #include "JITMulGenerator.h"
76 #include "JITRightShiftGenerator.h"
77 #include "JITSubGenerator.h"
78 #include "JSAsyncFunction.h"
79 #include "JSAsyncGeneratorFunction.h"
80 #include "JSCInlines.h"
81 #include "JSGeneratorFunction.h"
82 #include "JSImmutableButterfly.h"
83 #include "JSLexicalEnvironment.h"
84 #include "JSMap.h"
85 #include "OperandsInlines.h"
86 #include "ProbeContext.h"
87 #include "RegExpObject.h"
88 #include "ScopedArguments.h"
89 #include "ScopedArgumentsTable.h"
90 #include "ScratchRegisterAllocator.h"
91 #include "SetupVarargsFrame.h"
92 #include "ShadowChicken.h"
93 #include "StructureStubInfo.h"
94 #include "SuperSampler.h"
95 #include "ThunkGenerators.h"
96 #include "VirtualRegister.h"
97 #include "Watchdog.h"
98 #include <atomic>
99 #include <wtf/Box.h>
100 #include <wtf/Gigacage.h>
101 #include <wtf/RecursableLambda.h>
102 #include <wtf/StdUnorderedSet.h>
103
104 #undef RELEASE_ASSERT
105 #define RELEASE_ASSERT(assertion) do { \
106     if (!(assertion)) { \
107         WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #assertion); \
108         CRASH(); \
109     } \
110 } while (0)
111
112 namespace JSC { namespace FTL {
113
114 using namespace B3;
115 using namespace DFG;
116
117 namespace {
118
119 std::atomic<int> compileCounter;
120
121 #if !ASSERT_DISABLED
122 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
123     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
124 {
125     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
126     if (nodeIndex != UINT_MAX)
127         dataLog(", node @", nodeIndex);
128     dataLog(".\n");
129     CRASH();
130 }
131 #endif
132
133 // Using this instead of typeCheck() helps to reduce the load on B3, by creating
134 // significantly less dead code.
135 #define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \
136         FormattedValue _ftc_lowValue = (lowValue);                      \
137         Edge _ftc_highValue = (highValue);                              \
138         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
139         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
140             break;                                                      \
141         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \
142     } while (false)
143
144 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \
145     FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition)
146
147 class LowerDFGToB3 {
148     WTF_MAKE_NONCOPYABLE(LowerDFGToB3);
149 public:
150     LowerDFGToB3(State& state)
151         : m_graph(state.graph)
152         , m_ftlState(state)
153         , m_out(state)
154         , m_proc(*state.proc)
155         , m_availabilityCalculator(m_graph)
156         , m_state(state.graph)
157         , m_interpreter(state.graph, m_state)
158         , m_indexMaskingMode(Options::enableSpectreMitigations() ?  IndexMaskingEnabled : IndexMaskingDisabled)
159     {
160         if (Options::validateAbstractInterpreterState()) {
161             performLivenessAnalysis(m_graph);
162
163             // We only use node liveness here, not combined liveness, as we only track
164             // AI state for live nodes.
165             for (DFG::BasicBlock* block : m_graph.blocksInNaturalOrder()) {
166                 NodeSet live;
167
168                 for (NodeFlowProjection node : block->ssa->liveAtTail) {
169                     if (node.kind() == NodeFlowProjection::Primary)
170                         live.addVoid(node.node());
171                 }
172
173                 for (unsigned i = block->size(); i--; ) {
174                     Node* node = block->at(i);
175                     live.remove(node);
176                     m_graph.doToChildren(node, [&] (Edge child) {
177                         live.addVoid(child.node());
178                     });
179                     m_liveInToNode.add(node, live);
180                 }
181             }
182         }
183     }
184     
185     void lower()
186     {
187         State* state = &m_ftlState;
188         
189         CString name;
190         if (verboseCompilationEnabled()) {
191             name = toCString(
192                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
193                 "_", codeBlock()->hash());
194         } else
195             name = "jsBody";
196
197         {
198             m_proc.setNumEntrypoints(m_graph.m_numberOfEntrypoints);
199             CodeBlock* codeBlock = m_graph.m_codeBlock;
200
201             Ref<B3::Air::PrologueGenerator> catchPrologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>(
202                 [codeBlock] (CCallHelpers& jit, B3::Air::Code& code) {
203                     AllowMacroScratchRegisterUsage allowScratch(jit);
204                     jit.addPtr(CCallHelpers::TrustedImm32(-code.frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
205                     if (Options::zeroStackFrame())
206                         jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, code.frameSize());
207
208                     jit.emitSave(code.calleeSaveRegisterAtOffsetList());
209                     jit.emitPutToCallFrameHeader(codeBlock, CallFrameSlot::codeBlock);
210                 });
211
212             for (unsigned catchEntrypointIndex : m_graph.m_entrypointIndexToCatchBytecodeOffset.keys()) {
213                 RELEASE_ASSERT(catchEntrypointIndex != 0);
214                 m_proc.code().setPrologueForEntrypoint(catchEntrypointIndex, catchPrologueGenerator.copyRef());
215             }
216
217             if (m_graph.m_maxLocalsForCatchOSREntry) {
218                 uint32_t numberOfLiveLocals = std::max(*m_graph.m_maxLocalsForCatchOSREntry, 1u); // Make sure we always allocate a non-null catchOSREntryBuffer.
219                 m_ftlState.jitCode->common.catchOSREntryBuffer = m_graph.m_vm.scratchBufferForSize(sizeof(JSValue) * numberOfLiveLocals);
220             }
221         }
222         
223         m_graph.ensureSSADominators();
224
225         if (verboseCompilationEnabled())
226             dataLog("Function ready, beginning lowering.\n");
227
228         m_out.initialize(m_heaps);
229
230         // We use prologue frequency for all of the initialization code.
231         m_out.setFrequency(1);
232         
233         bool hasMultipleEntrypoints = m_graph.m_numberOfEntrypoints > 1;
234     
235         LBasicBlock prologue = m_out.newBlock();
236         LBasicBlock callEntrypointArgumentSpeculations = hasMultipleEntrypoints ? m_out.newBlock() : nullptr;
237         m_handleExceptions = m_out.newBlock();
238
239         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
240             m_highBlock = m_graph.block(blockIndex);
241             if (!m_highBlock)
242                 continue;
243             m_out.setFrequency(m_highBlock->executionCount);
244             m_blocks.add(m_highBlock, m_out.newBlock());
245         }
246
247         // Back to prologue frequency for any bocks that get sneakily created in the initialization code.
248         m_out.setFrequency(1);
249         
250         m_out.appendTo(prologue, hasMultipleEntrypoints ? callEntrypointArgumentSpeculations : m_handleExceptions);
251         m_out.initializeConstants(m_proc, prologue);
252         createPhiVariables();
253
254         size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;
255         B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);
256         m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));
257         state->capturedValue = capturedBase->slot();
258
259         auto preOrder = m_graph.blocksInPreOrder();
260
261         m_callFrame = m_out.framePointer();
262         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
263         m_tagMask = m_out.constInt64(TagMask);
264
265         // Make sure that B3 knows that we really care about the mask registers. This forces the
266         // constants to be materialized in registers.
267         m_proc.addFastConstant(m_tagTypeNumber->key());
268         m_proc.addFastConstant(m_tagMask->key());
269         
270         // We don't want the CodeBlock to have a weak pointer to itself because
271         // that would cause it to always get collected.
272         m_out.storePtr(m_out.constIntPtr(bitwise_cast<intptr_t>(codeBlock())), addressFor(CallFrameSlot::codeBlock));
273
274         VM* vm = &this->vm();
275
276         // Stack Overflow Check.
277         unsigned exitFrameSize = m_graph.requiredRegisterCountForExit() * sizeof(Register);
278         MacroAssembler::AbsoluteAddress addressOfStackLimit(vm->addressOfSoftStackLimit());
279         PatchpointValue* stackOverflowHandler = m_out.patchpoint(Void);
280         CallSiteIndex callSiteIndex = callSiteIndexForCodeOrigin(m_ftlState, CodeOrigin(0));
281         stackOverflowHandler->appendSomeRegister(m_callFrame);
282         stackOverflowHandler->clobber(RegisterSet::macroScratchRegisters());
283         stackOverflowHandler->numGPScratchRegisters = 1;
284         stackOverflowHandler->setGenerator(
285             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
286                 AllowMacroScratchRegisterUsage allowScratch(jit);
287                 GPRReg fp = params[0].gpr();
288                 GPRReg scratch = params.gpScratch(0);
289
290                 unsigned ftlFrameSize = params.proc().frameSize();
291                 unsigned maxFrameSize = std::max(exitFrameSize, ftlFrameSize);
292
293                 jit.addPtr(MacroAssembler::TrustedImm32(-maxFrameSize), fp, scratch);
294                 MacroAssembler::JumpList stackOverflow;
295                 if (UNLIKELY(maxFrameSize > Options::reservedZoneSize()))
296                     stackOverflow.append(jit.branchPtr(MacroAssembler::Above, scratch, fp));
297                 stackOverflow.append(jit.branchPtr(MacroAssembler::Above, addressOfStackLimit, scratch));
298
299                 params.addLatePath([=] (CCallHelpers& jit) {
300                     AllowMacroScratchRegisterUsage allowScratch(jit);
301
302                     stackOverflow.link(&jit);
303                     
304                     // FIXME: We would not have to do this if the stack check was part of the Air
305                     // prologue. Then, we would know that there is no way for the callee-saves to
306                     // get clobbered.
307                     // https://bugs.webkit.org/show_bug.cgi?id=172456
308                     jit.emitRestore(params.proc().calleeSaveRegisterAtOffsetList());
309                     
310                     jit.store32(
311                         MacroAssembler::TrustedImm32(callSiteIndex.bits()),
312                         CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
313                     jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm->topEntryFrame);
314
315                     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
316                     jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()), GPRInfo::argumentGPR1);
317                     CCallHelpers::Call throwCall = jit.call(OperationPtrTag);
318
319                     jit.move(CCallHelpers::TrustedImmPtr(vm), GPRInfo::argumentGPR0);
320                     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
321                     CCallHelpers::Call lookupExceptionHandlerCall = jit.call(OperationPtrTag);
322                     jit.jumpToExceptionHandler(*vm);
323
324                     jit.addLinkTask(
325                         [=] (LinkBuffer& linkBuffer) {
326                             linkBuffer.link(throwCall, FunctionPtr<OperationPtrTag>(operationThrowStackOverflowError));
327                             linkBuffer.link(lookupExceptionHandlerCall, FunctionPtr<OperationPtrTag>(lookupExceptionHandlerFromCallerFrame));
328                     });
329                 });
330             });
331
332         LBasicBlock firstDFGBasicBlock = lowBlock(m_graph.block(0));
333
334         {
335             if (hasMultipleEntrypoints) {
336                 Vector<LBasicBlock> successors(m_graph.m_numberOfEntrypoints);
337                 successors[0] = callEntrypointArgumentSpeculations;
338                 for (unsigned i = 1; i < m_graph.m_numberOfEntrypoints; ++i) {
339                     // Currently, the only other entrypoint is an op_catch entrypoint.
340                     // We do OSR entry at op_catch, and we prove argument formats before
341                     // jumping to FTL code, so we don't need to check argument types here
342                     // for these entrypoints.
343                     successors[i] = firstDFGBasicBlock;
344                 }
345                 
346                 m_out.entrySwitch(successors);
347                 m_out.appendTo(callEntrypointArgumentSpeculations, m_handleExceptions);
348             }
349
350             m_node = nullptr;
351             m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
352
353             // Check Arguments.
354             availabilityMap().clear();
355             availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
356             for (unsigned i = codeBlock()->numParameters(); i--;) {
357                 availabilityMap().m_locals.argument(i) =
358                     Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
359             }
360
361             for (unsigned i = codeBlock()->numParameters(); i--;) {
362                 MethodOfGettingAValueProfile profile(&m_graph.m_profiledBlock->valueProfileForArgument(i));
363                 VirtualRegister operand = virtualRegisterForArgument(i);
364                 LValue jsValue = m_out.load64(addressFor(operand));
365                 
366                 switch (m_graph.m_argumentFormats[0][i]) {
367                 case FlushedInt32:
368                     speculate(BadType, jsValueValue(jsValue), profile, isNotInt32(jsValue));
369                     break;
370                 case FlushedBoolean:
371                     speculate(BadType, jsValueValue(jsValue), profile, isNotBoolean(jsValue));
372                     break;
373                 case FlushedCell:
374                     speculate(BadType, jsValueValue(jsValue), profile, isNotCell(jsValue));
375                     break;
376                 case FlushedJSValue:
377                     break;
378                 default:
379                     DFG_CRASH(m_graph, nullptr, "Bad flush format for argument");
380                     break;
381                 }
382             }
383             m_out.jump(firstDFGBasicBlock);
384         }
385
386
387         m_out.appendTo(m_handleExceptions, firstDFGBasicBlock);
388         Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;
389         m_out.patchpoint(Void)->setGenerator(
390             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
391                 CCallHelpers::Jump jump = jit.jump();
392                 jit.addLinkTask(
393                     [=] (LinkBuffer& linkBuffer) {
394                         linkBuffer.link(jump, linkBuffer.locationOf<ExceptionHandlerPtrTag>(*exceptionHandler));
395                     });
396             });
397         m_out.unreachable();
398
399         for (DFG::BasicBlock* block : preOrder)
400             compileBlock(block);
401
402         // Make sure everything is decorated. This does a bunch of deferred decorating. This has
403         // to happen last because our abstract heaps are generated lazily. They have to be
404         // generated lazily because we have an infinite number of numbered, indexed, and
405         // absolute heaps. We only become aware of the ones we actually mention while lowering.
406         m_heaps.computeRangesAndDecorateInstructions();
407
408         // We create all Phi's up front, but we may then decide not to compile the basic block
409         // that would have contained one of them. So this creates orphans, which triggers B3
410         // validation failures. Calling this fixes the issue.
411         //
412         // Note that you should avoid the temptation to make this call conditional upon
413         // validation being enabled. B3 makes no guarantees of any kind of correctness when
414         // dealing with IR that would have failed validation. For example, it would be valid to
415         // write a B3 phase that so aggressively assumes the lack of orphans that it would crash
416         // if any orphans were around. We might even have such phases already.
417         m_proc.deleteOrphans();
418
419         // We put the blocks into the B3 procedure in a super weird order. Now we reorder them.
420         m_out.applyBlockOrder();
421     }
422
423 private:
424     
425     void createPhiVariables()
426     {
427         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
428             DFG::BasicBlock* block = m_graph.block(blockIndex);
429             if (!block)
430                 continue;
431             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
432                 Node* node = block->at(nodeIndex);
433                 if (node->op() != DFG::Phi)
434                     continue;
435                 LType type;
436                 switch (node->flags() & NodeResultMask) {
437                 case NodeResultDouble:
438                     type = Double;
439                     break;
440                 case NodeResultInt32:
441                     type = Int32;
442                     break;
443                 case NodeResultInt52:
444                     type = Int64;
445                     break;
446                 case NodeResultBoolean:
447                     type = Int32;
448                     break;
449                 case NodeResultJS:
450                     type = Int64;
451                     break;
452                 default:
453                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
454                     break;
455                 }
456                 m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));
457             }
458         }
459     }
460     
461     void compileBlock(DFG::BasicBlock* block)
462     {
463         if (!block)
464             return;
465         
466         if (verboseCompilationEnabled())
467             dataLog("Compiling block ", *block, "\n");
468         
469         m_highBlock = block;
470         
471         // Make sure that any blocks created while lowering code in the high block have the frequency of
472         // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs
473         // something roughly approximate for things like register allocation.
474         m_out.setFrequency(m_highBlock->executionCount);
475         
476         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
477         
478         m_nextHighBlock = 0;
479         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
480             m_nextHighBlock = m_graph.block(nextBlockIndex);
481             if (m_nextHighBlock)
482                 break;
483         }
484         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
485         
486         // All of this effort to find the next block gives us the ability to keep the
487         // generated IR in roughly program order. This ought not affect the performance
488         // of the generated code (since we expect B3 to reorder things) but it will
489         // make IR dumps easier to read.
490         m_out.appendTo(lowBlock, m_nextLowBlock);
491         
492         if (Options::ftlCrashes())
493             m_out.trap();
494         
495         if (!m_highBlock->cfaHasVisited) {
496             if (verboseCompilationEnabled())
497                 dataLog("Bailing because CFA didn't reach.\n");
498             crash(m_highBlock, nullptr);
499             return;
500         }
501
502         m_aiCheckedNodes.clear();
503         
504         m_availabilityCalculator.beginBlock(m_highBlock);
505         
506         m_state.reset();
507         m_state.beginBasicBlock(m_highBlock);
508         
509         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
510             if (!compileNode(m_nodeIndex))
511                 break;
512         }
513     }
514
515     void safelyInvalidateAfterTermination()
516     {
517         if (verboseCompilationEnabled())
518             dataLog("Bailing.\n");
519         crash();
520
521         // Invalidate dominated blocks. Under normal circumstances we would expect
522         // them to be invalidated already. But you can have the CFA become more
523         // precise over time because the structures of objects change on the main
524         // thread. Failing to do this would result in weird crashes due to a value
525         // being used but not defined. Race conditions FTW!
526         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
527             DFG::BasicBlock* target = m_graph.block(blockIndex);
528             if (!target)
529                 continue;
530             if (m_graph.m_ssaDominators->dominates(m_highBlock, target)) {
531                 if (verboseCompilationEnabled())
532                     dataLog("Block ", *target, " will bail also.\n");
533                 target->cfaHasVisited = false;
534             }
535         }
536     }
537
538     void validateAIState(Node* node)
539     {
540         if (!m_graphDump) {
541             StringPrintStream out;
542             m_graph.dump(out);
543             m_graphDump = out.toString();
544         }
545
546         switch (node->op()) {
547         case MovHint:
548         case ZombieHint:
549         case JSConstant:
550         case LazyJSConstant:
551         case DoubleConstant:
552         case Int52Constant:
553         case GetStack:
554         case PutStack:
555         case KillStack:
556         case ExitOK:
557             return;
558         default:
559             break;
560         }
561
562         // Before we execute node.
563         NodeSet& live = m_liveInToNode.find(node)->value;
564         unsigned highParentIndex = node->index();
565         {
566             uint64_t hash = WTF::intHash(highParentIndex);
567             if (hash >= static_cast<uint64_t>((static_cast<double>(std::numeric_limits<unsigned>::max()) + 1) * Options::validateAbstractInterpreterStateProbability()))
568                 return;
569         }
570
571         for (Node* node : live) {
572             if (node->isPhantomAllocation())
573                 continue;
574
575             if (node->op() == CheckInBounds)
576                 continue;
577
578             AbstractValue value = m_interpreter.forNode(node);
579             {
580                 auto iter = m_aiCheckedNodes.find(node);
581                 if (iter != m_aiCheckedNodes.end()) {
582                     AbstractValue checkedValue = iter->value;
583                     if (checkedValue == value) {
584                         if (!(value.m_type & SpecCell))
585                             continue;
586                     }
587                 }
588                 m_aiCheckedNodes.set(node, value);
589             }
590
591             FlushFormat flushFormat;
592             LValue input;
593             if (node->hasJSResult()) {
594                 input = lowJSValue(Edge(node, UntypedUse));
595                 flushFormat = FlushedJSValue;
596             } else if (node->hasDoubleResult()) {
597                 input = lowDouble(Edge(node, DoubleRepUse));
598                 flushFormat = FlushedDouble;
599             } else if (node->hasInt52Result()) {
600                 input = strictInt52ToJSValue(lowStrictInt52(Edge(node, Int52RepUse)));
601                 flushFormat = FlushedInt52;
602             } else
603                 continue;
604
605             unsigned highChildIndex = node->index();
606
607             String graphDump = m_graphDump;
608
609             PatchpointValue* patchpoint = m_out.patchpoint(Void);
610             patchpoint->effects = Effects::none();
611             patchpoint->effects.writesLocalState = true;
612             patchpoint->appendSomeRegister(input);
613             patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
614                 GPRReg reg = InvalidGPRReg;
615                 FPRReg fpReg = InvalidFPRReg;
616                 if (flushFormat == FlushedDouble)
617                     fpReg = params[0].fpr();
618                 else
619                     reg = params[0].gpr();
620                 jit.probe([=] (Probe::Context& context) {
621                     JSValue input;
622                     double doubleInput;
623
624                     auto dumpAndCrash = [&] {
625                         dataLogLn("Validation failed at node: @", highParentIndex);
626                         dataLogLn("Failed validating live value: @", highChildIndex);
627                         dataLogLn();
628                         dataLogLn("Expected AI value = ", value);
629                         if (flushFormat != FlushedDouble)
630                             dataLogLn("Unexpected value = ", input);
631                         else
632                             dataLogLn("Unexpected double value = ", doubleInput);
633                         dataLogLn();
634                         dataLogLn(graphDump);
635                         CRASH();
636                     };
637
638                     if (flushFormat == FlushedDouble) {
639                         doubleInput = context.fpr(fpReg);
640                         SpeculatedType type;
641                         if (!std::isnan(doubleInput))
642                             type = speculationFromValue(jsDoubleNumber(doubleInput));
643                         else if (isImpureNaN(doubleInput))
644                             type = SpecDoubleImpureNaN;
645                         else
646                             type = SpecDoublePureNaN;
647
648                         if (!value.couldBeType(type))
649                             dumpAndCrash();
650                     } else {
651                         input = JSValue::decode(context.gpr(reg)); 
652                         if (flushFormat == FlushedInt52) {
653                             RELEASE_ASSERT(input.isAnyInt());
654                             input = jsDoubleNumber(input.asAnyInt());
655                         }
656                         if (!value.validateOSREntryValue(input, flushFormat))
657                             dumpAndCrash();
658                     }
659
660                 });
661             });
662         }
663     }
664
665     bool compileNode(unsigned nodeIndex)
666     {
667         if (!m_state.isValid()) {
668             safelyInvalidateAfterTermination();
669             return false;
670         }
671         
672         m_node = m_highBlock->at(nodeIndex);
673         m_origin = m_node->origin;
674         m_out.setOrigin(m_node);
675         
676         if (verboseCompilationEnabled())
677             dataLog("Lowering ", m_node, "\n");
678         
679         m_interpreter.startExecuting();
680         m_interpreter.executeKnownEdgeTypes(m_node);
681
682         if (Options::validateAbstractInterpreterState())
683             validateAIState(m_node);
684
685         if (validateDFGDoesGC) {
686             bool expectDoesGC = doesGC(m_graph, m_node);
687             m_out.store(m_out.constBool(expectDoesGC), m_out.absolute(vm().heap.addressOfExpectDoesGC()));
688         }
689
690         switch (m_node->op()) {
691         case DFG::Upsilon:
692             compileUpsilon();
693             break;
694         case DFG::Phi:
695             compilePhi();
696             break;
697         case JSConstant:
698             break;
699         case DoubleConstant:
700             compileDoubleConstant();
701             break;
702         case Int52Constant:
703             compileInt52Constant();
704             break;
705         case LazyJSConstant:
706             compileLazyJSConstant();
707             break;
708         case DoubleRep:
709             compileDoubleRep();
710             break;
711         case DoubleAsInt32:
712             compileDoubleAsInt32();
713             break;
714         case DFG::ValueRep:
715             compileValueRep();
716             break;
717         case Int52Rep:
718             compileInt52Rep();
719             break;
720         case ValueToInt32:
721             compileValueToInt32();
722             break;
723         case BooleanToNumber:
724             compileBooleanToNumber();
725             break;
726         case ExtractOSREntryLocal:
727             compileExtractOSREntryLocal();
728             break;
729         case ExtractCatchLocal:
730             compileExtractCatchLocal();
731             break;
732         case ClearCatchLocals:
733             compileClearCatchLocals();
734             break;
735         case GetStack:
736             compileGetStack();
737             break;
738         case PutStack:
739             compilePutStack();
740             break;
741         case DFG::Check:
742         case CheckVarargs:
743             compileNoOp();
744             break;
745         case ToObject:
746         case CallObjectConstructor:
747             compileToObjectOrCallObjectConstructor();
748             break;
749         case ToThis:
750             compileToThis();
751             break;
752         case ValueNegate:
753             compileValueNegate();
754             break;
755         case ValueAdd:
756             compileValueAdd();
757             break;
758         case ValueSub:
759             compileValueSub();
760             break;
761         case ValueMul:
762             compileValueMul();
763             break;
764         case StrCat:
765             compileStrCat();
766             break;
767         case ArithAdd:
768         case ArithSub:
769             compileArithAddOrSub();
770             break;
771         case ArithClz32:
772             compileArithClz32();
773             break;
774         case ArithMul:
775             compileArithMul();
776             break;
777         case ValueDiv:
778             compileValueDiv();
779             break;
780         case ArithDiv:
781             compileArithDiv();
782             break;
783         case ValueMod:
784             compileValueMod();
785             break;
786         case ArithMod:
787             compileArithMod();
788             break;
789         case ArithMin:
790         case ArithMax:
791             compileArithMinOrMax();
792             break;
793         case ArithAbs:
794             compileArithAbs();
795             break;
796         case ValuePow:
797             compileValuePow();
798             break;
799         case ArithPow:
800             compileArithPow();
801             break;
802         case ArithRandom:
803             compileArithRandom();
804             break;
805         case ArithRound:
806             compileArithRound();
807             break;
808         case ArithFloor:
809             compileArithFloor();
810             break;
811         case ArithCeil:
812             compileArithCeil();
813             break;
814         case ArithTrunc:
815             compileArithTrunc();
816             break;
817         case ArithSqrt:
818             compileArithSqrt();
819             break;
820         case ArithFRound:
821             compileArithFRound();
822             break;
823         case ArithNegate:
824             compileArithNegate();
825             break;
826         case ArithUnary:
827             compileArithUnary();
828             break;
829         case ValueBitNot:
830             compileValueBitNot();
831             break;
832         case ArithBitNot:
833             compileArithBitNot();
834             break;
835         case ValueBitAnd:
836             compileValueBitAnd();
837             break;
838         case ArithBitAnd:
839             compileArithBitAnd();
840             break;
841         case ValueBitOr:
842             compileValueBitOr();
843             break;
844         case ArithBitOr:
845             compileArithBitOr();
846             break;
847         case ArithBitXor:
848             compileArithBitXor();
849             break;
850         case ValueBitXor:
851             compileValueBitXor();
852             break;
853         case BitRShift:
854             compileBitRShift();
855             break;
856         case ArithBitLShift:
857             compileArithBitLShift();
858             break;
859         case ValueBitLShift:
860             compileValueBitLShift();
861             break;
862         case BitURShift:
863             compileBitURShift();
864             break;
865         case UInt32ToNumber:
866             compileUInt32ToNumber();
867             break;
868         case CheckStructure:
869             compileCheckStructure();
870             break;
871         case CheckStructureOrEmpty:
872             compileCheckStructureOrEmpty();
873             break;
874         case CheckCell:
875             compileCheckCell();
876             break;
877         case CheckNotEmpty:
878             compileCheckNotEmpty();
879             break;
880         case AssertNotEmpty:
881             compileAssertNotEmpty();
882             break;
883         case CheckBadCell:
884             compileCheckBadCell();
885             break;
886         case CheckStringIdent:
887             compileCheckStringIdent();
888             break;
889         case GetExecutable:
890             compileGetExecutable();
891             break;
892         case Arrayify:
893         case ArrayifyToStructure:
894             compileArrayify();
895             break;
896         case PutStructure:
897             compilePutStructure();
898             break;
899         case TryGetById:
900             compileGetById(AccessType::TryGet);
901             break;
902         case GetById:
903         case GetByIdFlush:
904             compileGetById(AccessType::Get);
905             break;
906         case GetByIdWithThis:
907             compileGetByIdWithThis();
908             break;
909         case GetByIdDirect:
910         case GetByIdDirectFlush:
911             compileGetById(AccessType::GetDirect);
912             break;
913         case InById:
914             compileInById();
915             break;
916         case InByVal:
917             compileInByVal();
918             break;
919         case HasOwnProperty:
920             compileHasOwnProperty();
921             break;
922         case PutById:
923         case PutByIdDirect:
924         case PutByIdFlush:
925             compilePutById();
926             break;
927         case PutByIdWithThis:
928             compilePutByIdWithThis();
929             break;
930         case PutGetterById:
931         case PutSetterById:
932             compilePutAccessorById();
933             break;
934         case PutGetterSetterById:
935             compilePutGetterSetterById();
936             break;
937         case PutGetterByVal:
938         case PutSetterByVal:
939             compilePutAccessorByVal();
940             break;
941         case DeleteById:
942             compileDeleteById();
943             break;
944         case DeleteByVal:
945             compileDeleteByVal();
946             break;
947         case GetButterfly:
948             compileGetButterfly();
949             break;
950         case ConstantStoragePointer:
951             compileConstantStoragePointer();
952             break;
953         case GetIndexedPropertyStorage:
954             compileGetIndexedPropertyStorage();
955             break;
956         case CheckArray:
957             compileCheckArray();
958             break;
959         case GetArrayLength:
960             compileGetArrayLength();
961             break;
962         case GetVectorLength:
963             compileGetVectorLength();
964             break;
965         case CheckInBounds:
966             compileCheckInBounds();
967             break;
968         case GetByVal:
969             compileGetByVal();
970             break;
971         case GetMyArgumentByVal:
972         case GetMyArgumentByValOutOfBounds:
973             compileGetMyArgumentByVal();
974             break;
975         case GetByValWithThis:
976             compileGetByValWithThis();
977             break;
978         case PutByVal:
979         case PutByValAlias:
980         case PutByValDirect:
981             compilePutByVal();
982             break;
983         case PutByValWithThis:
984             compilePutByValWithThis();
985             break;
986         case AtomicsAdd:
987         case AtomicsAnd:
988         case AtomicsCompareExchange:
989         case AtomicsExchange:
990         case AtomicsLoad:
991         case AtomicsOr:
992         case AtomicsStore:
993         case AtomicsSub:
994         case AtomicsXor:
995             compileAtomicsReadModifyWrite();
996             break;
997         case AtomicsIsLockFree:
998             compileAtomicsIsLockFree();
999             break;
1000         case DefineDataProperty:
1001             compileDefineDataProperty();
1002             break;
1003         case DefineAccessorProperty:
1004             compileDefineAccessorProperty();
1005             break;
1006         case ArrayPush:
1007             compileArrayPush();
1008             break;
1009         case ArrayPop:
1010             compileArrayPop();
1011             break;
1012         case ArraySlice:
1013             compileArraySlice();
1014             break;
1015         case ArrayIndexOf:
1016             compileArrayIndexOf();
1017             break;
1018         case CreateActivation:
1019             compileCreateActivation();
1020             break;
1021         case PushWithScope:
1022             compilePushWithScope();
1023             break;
1024         case NewFunction:
1025         case NewGeneratorFunction:
1026         case NewAsyncGeneratorFunction:
1027         case NewAsyncFunction:
1028             compileNewFunction();
1029             break;
1030         case CreateDirectArguments:
1031             compileCreateDirectArguments();
1032             break;
1033         case CreateScopedArguments:
1034             compileCreateScopedArguments();
1035             break;
1036         case CreateClonedArguments:
1037             compileCreateClonedArguments();
1038             break;
1039         case ObjectCreate:
1040             compileObjectCreate();
1041             break;
1042         case ObjectKeys:
1043             compileObjectKeys();
1044             break;
1045         case NewObject:
1046             compileNewObject();
1047             break;
1048         case NewPromise:
1049             compileNewPromise();
1050             break;
1051         case NewStringObject:
1052             compileNewStringObject();
1053             break;
1054         case NewSymbol:
1055             compileNewSymbol();
1056             break;
1057         case NewArray:
1058             compileNewArray();
1059             break;
1060         case NewArrayWithSpread:
1061             compileNewArrayWithSpread();
1062             break;
1063         case CreateThis:
1064             compileCreateThis();
1065             break;
1066         case CreatePromise:
1067             compileCreatePromise();
1068             break;
1069         case Spread:
1070             compileSpread();
1071             break;
1072         case NewArrayBuffer:
1073             compileNewArrayBuffer();
1074             break;
1075         case NewArrayWithSize:
1076             compileNewArrayWithSize();
1077             break;
1078         case NewTypedArray:
1079             compileNewTypedArray();
1080             break;
1081         case GetTypedArrayByteOffset:
1082             compileGetTypedArrayByteOffset();
1083             break;
1084         case GetPrototypeOf:
1085             compileGetPrototypeOf();
1086             break;
1087         case AllocatePropertyStorage:
1088             compileAllocatePropertyStorage();
1089             break;
1090         case ReallocatePropertyStorage:
1091             compileReallocatePropertyStorage();
1092             break;
1093         case NukeStructureAndSetButterfly:
1094             compileNukeStructureAndSetButterfly();
1095             break;
1096         case ToNumber:
1097             compileToNumber();
1098             break;
1099         case ToString:
1100         case CallStringConstructor:
1101         case StringValueOf:
1102             compileToStringOrCallStringConstructorOrStringValueOf();
1103             break;
1104         case ToPrimitive:
1105             compileToPrimitive();
1106             break;
1107         case MakeRope:
1108             compileMakeRope();
1109             break;
1110         case StringCharAt:
1111             compileStringCharAt();
1112             break;
1113         case StringCharCodeAt:
1114             compileStringCharCodeAt();
1115             break;
1116         case StringFromCharCode:
1117             compileStringFromCharCode();
1118             break;
1119         case GetByOffset:
1120         case GetGetterSetterByOffset:
1121             compileGetByOffset();
1122             break;
1123         case GetGetter:
1124             compileGetGetter();
1125             break;
1126         case GetSetter:
1127             compileGetSetter();
1128             break;
1129         case MultiGetByOffset:
1130             compileMultiGetByOffset();
1131             break;
1132         case PutByOffset:
1133             compilePutByOffset();
1134             break;
1135         case MultiPutByOffset:
1136             compileMultiPutByOffset();
1137             break;
1138         case MatchStructure:
1139             compileMatchStructure();
1140             break;
1141         case GetGlobalVar:
1142         case GetGlobalLexicalVariable:
1143             compileGetGlobalVariable();
1144             break;
1145         case PutGlobalVariable:
1146             compilePutGlobalVariable();
1147             break;
1148         case NotifyWrite:
1149             compileNotifyWrite();
1150             break;
1151         case GetCallee:
1152             compileGetCallee();
1153             break;
1154         case SetCallee:
1155             compileSetCallee();
1156             break;
1157         case GetArgumentCountIncludingThis:
1158             compileGetArgumentCountIncludingThis();
1159             break;
1160         case SetArgumentCountIncludingThis:
1161             compileSetArgumentCountIncludingThis();
1162             break;
1163         case GetScope:
1164             compileGetScope();
1165             break;
1166         case SkipScope:
1167             compileSkipScope();
1168             break;
1169         case GetGlobalObject:
1170             compileGetGlobalObject();
1171             break;
1172         case GetGlobalThis:
1173             compileGetGlobalThis();
1174             break;
1175         case GetClosureVar:
1176             compileGetClosureVar();
1177             break;
1178         case PutClosureVar:
1179             compilePutClosureVar();
1180             break;
1181         case GetPromiseInternalField:
1182             compileGetPromiseInternalField();
1183             break;
1184         case PutPromiseInternalField:
1185             compilePutPromiseInternalField();
1186             break;
1187         case GetFromArguments:
1188             compileGetFromArguments();
1189             break;
1190         case PutToArguments:
1191             compilePutToArguments();
1192             break;
1193         case GetArgument:
1194             compileGetArgument();
1195             break;
1196         case CompareEq:
1197             compileCompareEq();
1198             break;
1199         case CompareStrictEq:
1200             compileCompareStrictEq();
1201             break;
1202         case CompareLess:
1203             compileCompareLess();
1204             break;
1205         case CompareLessEq:
1206             compileCompareLessEq();
1207             break;
1208         case CompareGreater:
1209             compileCompareGreater();
1210             break;
1211         case CompareGreaterEq:
1212             compileCompareGreaterEq();
1213             break;
1214         case CompareBelow:
1215             compileCompareBelow();
1216             break;
1217         case CompareBelowEq:
1218             compileCompareBelowEq();
1219             break;
1220         case CompareEqPtr:
1221             compileCompareEqPtr();
1222             break;
1223         case SameValue:
1224             compileSameValue();
1225             break;
1226         case LogicalNot:
1227             compileLogicalNot();
1228             break;
1229         case Call:
1230         case TailCallInlinedCaller:
1231         case Construct:
1232             compileCallOrConstruct();
1233             break;
1234         case DirectCall:
1235         case DirectTailCallInlinedCaller:
1236         case DirectConstruct:
1237         case DirectTailCall:
1238             compileDirectCallOrConstruct();
1239             break;
1240         case TailCall:
1241             compileTailCall();
1242             break;
1243         case CallVarargs:
1244         case CallForwardVarargs:
1245         case TailCallVarargs:
1246         case TailCallVarargsInlinedCaller:
1247         case TailCallForwardVarargs:
1248         case TailCallForwardVarargsInlinedCaller:
1249         case ConstructVarargs:
1250         case ConstructForwardVarargs:
1251             compileCallOrConstructVarargs();
1252             break;
1253         case CallEval:
1254             compileCallEval();
1255             break;
1256         case LoadVarargs:
1257             compileLoadVarargs();
1258             break;
1259         case ForwardVarargs:
1260             compileForwardVarargs();
1261             break;
1262         case DFG::Jump:
1263             compileJump();
1264             break;
1265         case DFG::Branch:
1266             compileBranch();
1267             break;
1268         case DFG::Switch:
1269             compileSwitch();
1270             break;
1271         case DFG::EntrySwitch:
1272             compileEntrySwitch();
1273             break;
1274         case DFG::Return:
1275             compileReturn();
1276             break;
1277         case ForceOSRExit:
1278             compileForceOSRExit();
1279             break;
1280         case CPUIntrinsic:
1281 #if CPU(X86_64)
1282             compileCPUIntrinsic();
1283 #else
1284             RELEASE_ASSERT_NOT_REACHED();
1285 #endif
1286             break;
1287         case Throw:
1288             compileThrow();
1289             break;
1290         case ThrowStaticError:
1291             compileThrowStaticError();
1292             break;
1293         case InvalidationPoint:
1294             compileInvalidationPoint();
1295             break;
1296         case IsEmpty:
1297             compileIsEmpty();
1298             break;
1299         case IsUndefined:
1300             compileIsUndefined();
1301             break;
1302         case IsUndefinedOrNull:
1303             compileIsUndefinedOrNull();
1304             break;
1305         case IsBoolean:
1306             compileIsBoolean();
1307             break;
1308         case IsNumber:
1309             compileIsNumber();
1310             break;
1311         case NumberIsInteger:
1312             compileNumberIsInteger();
1313             break;
1314         case IsCellWithType:
1315             compileIsCellWithType();
1316             break;
1317         case MapHash:
1318             compileMapHash();
1319             break;
1320         case NormalizeMapKey:
1321             compileNormalizeMapKey();
1322             break;
1323         case GetMapBucket:
1324             compileGetMapBucket();
1325             break;
1326         case GetMapBucketHead:
1327             compileGetMapBucketHead();
1328             break;
1329         case GetMapBucketNext:
1330             compileGetMapBucketNext();
1331             break;
1332         case LoadKeyFromMapBucket:
1333             compileLoadKeyFromMapBucket();
1334             break;
1335         case LoadValueFromMapBucket:
1336             compileLoadValueFromMapBucket();
1337             break;
1338         case ExtractValueFromWeakMapGet:
1339             compileExtractValueFromWeakMapGet();
1340             break;
1341         case SetAdd:
1342             compileSetAdd();
1343             break;
1344         case MapSet:
1345             compileMapSet();
1346             break;
1347         case WeakMapGet:
1348             compileWeakMapGet();
1349             break;
1350         case WeakSetAdd:
1351             compileWeakSetAdd();
1352             break;
1353         case WeakMapSet:
1354             compileWeakMapSet();
1355             break;
1356         case IsObject:
1357             compileIsObject();
1358             break;
1359         case IsObjectOrNull:
1360             compileIsObjectOrNull();
1361             break;
1362         case IsFunction:
1363             compileIsFunction();
1364             break;
1365         case IsTypedArrayView:
1366             compileIsTypedArrayView();
1367             break;
1368         case ParseInt:
1369             compileParseInt();
1370             break;
1371         case TypeOf:
1372             compileTypeOf();
1373             break;
1374         case CheckTypeInfoFlags:
1375             compileCheckTypeInfoFlags();
1376             break;
1377         case OverridesHasInstance:
1378             compileOverridesHasInstance();
1379             break;
1380         case InstanceOf:
1381             compileInstanceOf();
1382             break;
1383         case InstanceOfCustom:
1384             compileInstanceOfCustom();
1385             break;
1386         case CountExecution:
1387             compileCountExecution();
1388             break;
1389         case SuperSamplerBegin:
1390             compileSuperSamplerBegin();
1391             break;
1392         case SuperSamplerEnd:
1393             compileSuperSamplerEnd();
1394             break;
1395         case StoreBarrier:
1396         case FencedStoreBarrier:
1397             compileStoreBarrier();
1398             break;
1399         case HasIndexedProperty:
1400             compileHasIndexedProperty();
1401             break;
1402         case HasGenericProperty:
1403             compileHasGenericProperty();
1404             break;
1405         case HasStructureProperty:
1406             compileHasStructureProperty();
1407             break;
1408         case GetDirectPname:
1409             compileGetDirectPname();
1410             break;
1411         case GetEnumerableLength:
1412             compileGetEnumerableLength();
1413             break;
1414         case GetPropertyEnumerator:
1415             compileGetPropertyEnumerator();
1416             break;
1417         case GetEnumeratorStructurePname:
1418             compileGetEnumeratorStructurePname();
1419             break;
1420         case GetEnumeratorGenericPname:
1421             compileGetEnumeratorGenericPname();
1422             break;
1423         case ToIndexString:
1424             compileToIndexString();
1425             break;
1426         case CheckStructureImmediate:
1427             compileCheckStructureImmediate();
1428             break;
1429         case MaterializeNewObject:
1430             compileMaterializeNewObject();
1431             break;
1432         case MaterializeCreateActivation:
1433             compileMaterializeCreateActivation();
1434             break;
1435         case CheckTraps:
1436             compileCheckTraps();
1437             break;
1438         case CreateRest:
1439             compileCreateRest();
1440             break;
1441         case GetRestLength:
1442             compileGetRestLength();
1443             break;
1444         case RegExpExec:
1445             compileRegExpExec();
1446             break;
1447         case RegExpExecNonGlobalOrSticky:
1448             compileRegExpExecNonGlobalOrSticky();
1449             break;
1450         case RegExpTest:
1451             compileRegExpTest();
1452             break;
1453         case RegExpMatchFast:
1454             compileRegExpMatchFast();
1455             break;
1456         case RegExpMatchFastGlobal:
1457             compileRegExpMatchFastGlobal();
1458             break;
1459         case NewRegexp:
1460             compileNewRegexp();
1461             break;
1462         case SetFunctionName:
1463             compileSetFunctionName();
1464             break;
1465         case StringReplace:
1466         case StringReplaceRegExp:
1467             compileStringReplace();
1468             break;
1469         case GetRegExpObjectLastIndex:
1470             compileGetRegExpObjectLastIndex();
1471             break;
1472         case SetRegExpObjectLastIndex:
1473             compileSetRegExpObjectLastIndex();
1474             break;
1475         case LogShadowChickenPrologue:
1476             compileLogShadowChickenPrologue();
1477             break;
1478         case LogShadowChickenTail:
1479             compileLogShadowChickenTail();
1480             break;
1481         case RecordRegExpCachedResult:
1482             compileRecordRegExpCachedResult();
1483             break;
1484         case ResolveScopeForHoistingFuncDeclInEval:
1485             compileResolveScopeForHoistingFuncDeclInEval();
1486             break;
1487         case ResolveScope:
1488             compileResolveScope();
1489             break;
1490         case GetDynamicVar:
1491             compileGetDynamicVar();
1492             break;
1493         case PutDynamicVar:
1494             compilePutDynamicVar();
1495             break;
1496         case Unreachable:
1497             compileUnreachable();
1498             break;
1499         case StringSlice:
1500             compileStringSlice();
1501             break;
1502         case ToLowerCase:
1503             compileToLowerCase();
1504             break;
1505         case NumberToStringWithRadix:
1506             compileNumberToStringWithRadix();
1507             break;
1508         case NumberToStringWithValidRadixConstant:
1509             compileNumberToStringWithValidRadixConstant();
1510             break;
1511         case CheckSubClass:
1512             compileCheckSubClass();
1513             break;
1514         case CallDOM:
1515             compileCallDOM();
1516             break;
1517         case CallDOMGetter:
1518             compileCallDOMGetter();
1519             break;
1520         case FilterCallLinkStatus:
1521         case FilterGetByIdStatus:
1522         case FilterPutByIdStatus:
1523         case FilterInByIdStatus:
1524             compileFilterICStatus();
1525             break;
1526         case DataViewGetInt:
1527         case DataViewGetFloat:
1528             compileDataViewGet();
1529             break;
1530         case DataViewSet:
1531             compileDataViewSet();
1532             break;
1533
1534         case PhantomLocal:
1535         case LoopHint:
1536         case MovHint:
1537         case ZombieHint:
1538         case ExitOK:
1539         case PhantomNewObject:
1540         case PhantomNewFunction:
1541         case PhantomNewGeneratorFunction:
1542         case PhantomNewAsyncGeneratorFunction:
1543         case PhantomNewAsyncFunction:
1544         case PhantomCreateActivation:
1545         case PhantomDirectArguments:
1546         case PhantomCreateRest:
1547         case PhantomSpread:
1548         case PhantomNewArrayWithSpread:
1549         case PhantomNewArrayBuffer:
1550         case PhantomClonedArguments:
1551         case PhantomNewRegexp:
1552         case PutHint:
1553         case BottomValue:
1554         case KillStack:
1555         case InitializeEntrypointArguments:
1556             break;
1557         default:
1558             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
1559             break;
1560         }
1561         
1562         if (m_node->isTerminal())
1563             return false;
1564         
1565         if (!m_state.isValid()) {
1566             safelyInvalidateAfterTermination();
1567             return false;
1568         }
1569
1570         m_availabilityCalculator.executeNode(m_node);
1571         m_interpreter.executeEffects(nodeIndex);
1572         
1573         return true;
1574     }
1575
1576     void compileUpsilon()
1577     {
1578         LValue upsilonValue = nullptr;
1579         switch (m_node->child1().useKind()) {
1580         case DoubleRepUse:
1581             upsilonValue = lowDouble(m_node->child1());
1582             break;
1583         case Int32Use:
1584         case KnownInt32Use:
1585             upsilonValue = lowInt32(m_node->child1());
1586             break;
1587         case Int52RepUse:
1588             upsilonValue = lowInt52(m_node->child1());
1589             break;
1590         case BooleanUse:
1591         case KnownBooleanUse:
1592             upsilonValue = lowBoolean(m_node->child1());
1593             break;
1594         case CellUse:
1595         case KnownCellUse:
1596             upsilonValue = lowCell(m_node->child1());
1597             break;
1598         case UntypedUse:
1599             upsilonValue = lowJSValue(m_node->child1());
1600             break;
1601         default:
1602             DFG_CRASH(m_graph, m_node, "Bad use kind");
1603             break;
1604         }
1605         ValueFromBlock upsilon = m_out.anchor(upsilonValue);
1606         LValue phiNode = m_phis.get(m_node->phi());
1607         m_out.addIncomingToPhi(phiNode, upsilon);
1608     }
1609     
1610     void compilePhi()
1611     {
1612         LValue phi = m_phis.get(m_node);
1613         m_out.m_block->append(phi);
1614
1615         switch (m_node->flags() & NodeResultMask) {
1616         case NodeResultDouble:
1617             setDouble(phi);
1618             break;
1619         case NodeResultInt32:
1620             setInt32(phi);
1621             break;
1622         case NodeResultInt52:
1623             setInt52(phi);
1624             break;
1625         case NodeResultBoolean:
1626             setBoolean(phi);
1627             break;
1628         case NodeResultJS:
1629             setJSValue(phi);
1630             break;
1631         default:
1632             DFG_CRASH(m_graph, m_node, "Bad result type");
1633             break;
1634         }
1635     }
1636     
1637     void compileDoubleConstant()
1638     {
1639         setDouble(m_out.constDouble(m_node->asNumber()));
1640     }
1641     
1642     void compileInt52Constant()
1643     {
1644         int64_t value = m_node->asAnyInt();
1645         
1646         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
1647         setStrictInt52(m_out.constInt64(value));
1648     }
1649
1650     void compileLazyJSConstant()
1651     {
1652         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
1653         LazyJSValue value = m_node->lazyJSValue();
1654         patchpoint->setGenerator(
1655             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1656                 value.emit(jit, JSValueRegs(params[0].gpr()));
1657             });
1658         patchpoint->effects = Effects::none();
1659         setJSValue(patchpoint);
1660     }
1661
1662     void compileDoubleRep()
1663     {
1664         switch (m_node->child1().useKind()) {
1665         case RealNumberUse: {
1666             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1667             
1668             LValue doubleValue = unboxDouble(value);
1669             
1670             LBasicBlock intCase = m_out.newBlock();
1671             LBasicBlock continuation = m_out.newBlock();
1672             
1673             ValueFromBlock fastResult = m_out.anchor(doubleValue);
1674             m_out.branch(
1675                 m_out.doubleEqual(doubleValue, doubleValue),
1676                 usually(continuation), rarely(intCase));
1677             
1678             LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
1679
1680             FTL_TYPE_CHECK(
1681                 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
1682                 isNotInt32(value, provenType(m_node->child1()) & ~SpecDoubleReal));
1683             ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
1684             m_out.jump(continuation);
1685             
1686             m_out.appendTo(continuation, lastNext);
1687             
1688             setDouble(m_out.phi(Double, fastResult, slowResult));
1689             return;
1690         }
1691             
1692         case NotCellUse:
1693         case NumberUse: {
1694             bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
1695             
1696             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1697
1698             LBasicBlock intCase = m_out.newBlock();
1699             LBasicBlock doubleTesting = m_out.newBlock();
1700             LBasicBlock doubleCase = m_out.newBlock();
1701             LBasicBlock nonDoubleCase = m_out.newBlock();
1702             LBasicBlock continuation = m_out.newBlock();
1703             
1704             m_out.branch(
1705                 isNotInt32(value, provenType(m_node->child1())),
1706                 unsure(doubleTesting), unsure(intCase));
1707             
1708             LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1709             
1710             ValueFromBlock intToDouble = m_out.anchor(
1711                 m_out.intToDouble(unboxInt32(value)));
1712             m_out.jump(continuation);
1713             
1714             m_out.appendTo(doubleTesting, doubleCase);
1715             LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1716             m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1717
1718             m_out.appendTo(doubleCase, nonDoubleCase);
1719             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1720             m_out.jump(continuation);
1721
1722             if (shouldConvertNonNumber) {
1723                 LBasicBlock undefinedCase = m_out.newBlock();
1724                 LBasicBlock testNullCase = m_out.newBlock();
1725                 LBasicBlock nullCase = m_out.newBlock();
1726                 LBasicBlock testBooleanTrueCase = m_out.newBlock();
1727                 LBasicBlock convertBooleanTrueCase = m_out.newBlock();
1728                 LBasicBlock convertBooleanFalseCase = m_out.newBlock();
1729
1730                 m_out.appendTo(nonDoubleCase, undefinedCase);
1731                 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1732                 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1733
1734                 m_out.appendTo(undefinedCase, testNullCase);
1735                 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1736                 m_out.jump(continuation);
1737
1738                 m_out.appendTo(testNullCase, nullCase);
1739                 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1740                 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1741
1742                 m_out.appendTo(nullCase, testBooleanTrueCase);
1743                 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1744                 m_out.jump(continuation);
1745
1746                 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1747                 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1748                 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1749
1750                 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1751                 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1752                 m_out.jump(continuation);
1753
1754                 m_out.appendTo(convertBooleanFalseCase, continuation);
1755
1756                 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1757                 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCellCheck, valueIsNotBooleanFalse);
1758                 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1759                 m_out.jump(continuation);
1760
1761                 m_out.appendTo(continuation, lastNext);
1762                 setDouble(m_out.phi(Double, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1763                 return;
1764             }
1765             m_out.appendTo(nonDoubleCase, continuation);
1766             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1767             m_out.unreachable();
1768
1769             m_out.appendTo(continuation, lastNext);
1770
1771             setDouble(m_out.phi(Double, intToDouble, unboxedDouble));
1772             return;
1773         }
1774             
1775         case Int52RepUse: {
1776             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1777             return;
1778         }
1779             
1780         default:
1781             DFG_CRASH(m_graph, m_node, "Bad use kind");
1782         }
1783     }
1784
1785     void compileDoubleAsInt32()
1786     {
1787         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1788         setInt32(integerValue);
1789     }
1790
1791     void compileValueRep()
1792     {
1793         switch (m_node->child1().useKind()) {
1794         case DoubleRepUse: {
1795             LValue value = lowDouble(m_node->child1());
1796             
1797             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1798                 value = m_out.select(
1799                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1800             }
1801             
1802             setJSValue(boxDouble(value));
1803             return;
1804         }
1805             
1806         case Int52RepUse: {
1807             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1808             return;
1809         }
1810             
1811         default:
1812             DFG_CRASH(m_graph, m_node, "Bad use kind");
1813         }
1814     }
1815     
1816     void compileInt52Rep()
1817     {
1818         switch (m_node->child1().useKind()) {
1819         case Int32Use:
1820             setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1())));
1821             return;
1822             
1823         case AnyIntUse:
1824             setStrictInt52(
1825                 jsValueToStrictInt52(
1826                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1827             return;
1828             
1829         case DoubleRepAnyIntUse:
1830             setStrictInt52(
1831                 doubleToStrictInt52(
1832                     m_node->child1(), lowDouble(m_node->child1())));
1833             return;
1834             
1835         default:
1836             RELEASE_ASSERT_NOT_REACHED();
1837         }
1838     }
1839     
1840     void compileValueToInt32()
1841     {
1842         switch (m_node->child1().useKind()) {
1843         case Int52RepUse:
1844             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1845             break;
1846             
1847         case DoubleRepUse:
1848             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1849             break;
1850             
1851         case NumberUse:
1852         case NotCellUse: {
1853             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1854             if (isValid(value)) {
1855                 setInt32(value.value());
1856                 break;
1857             }
1858             
1859             value = m_jsValueValues.get(m_node->child1().node());
1860             if (isValid(value)) {
1861                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1862                 break;
1863             }
1864             
1865             // We'll basically just get here for constants. But it's good to have this
1866             // catch-all since we often add new representations into the mix.
1867             setInt32(
1868                 numberOrNotCellToInt32(
1869                     m_node->child1(),
1870                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1871             break;
1872         }
1873             
1874         default:
1875             DFG_CRASH(m_graph, m_node, "Bad use kind");
1876             break;
1877         }
1878     }
1879     
1880     void compileBooleanToNumber()
1881     {
1882         switch (m_node->child1().useKind()) {
1883         case BooleanUse: {
1884             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), Int32));
1885             return;
1886         }
1887             
1888         case UntypedUse: {
1889             LValue value = lowJSValue(m_node->child1());
1890             
1891             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1892                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1893                 return;
1894             }
1895             
1896             LBasicBlock booleanCase = m_out.newBlock();
1897             LBasicBlock continuation = m_out.newBlock();
1898             
1899             ValueFromBlock notBooleanResult = m_out.anchor(value);
1900             m_out.branch(
1901                 isBoolean(value, provenType(m_node->child1())),
1902                 unsure(booleanCase), unsure(continuation));
1903             
1904             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1905             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1906                 m_out.zeroExt(unboxBoolean(value), Int64), m_tagTypeNumber));
1907             m_out.jump(continuation);
1908             
1909             m_out.appendTo(continuation, lastNext);
1910             setJSValue(m_out.phi(Int64, booleanResult, notBooleanResult));
1911             return;
1912         }
1913             
1914         default:
1915             RELEASE_ASSERT_NOT_REACHED();
1916             return;
1917         }
1918     }
1919
1920     void compileExtractOSREntryLocal()
1921     {
1922         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1923             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1924         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1925     }
1926
1927     void compileExtractCatchLocal()
1928     {
1929         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(m_ftlState.jitCode->common.catchOSREntryBuffer->dataBuffer());
1930         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->catchOSREntryIndex())));
1931     }
1932
1933     void compileClearCatchLocals()
1934     {
1935         ScratchBuffer* scratchBuffer = m_ftlState.jitCode->common.catchOSREntryBuffer;
1936         ASSERT(scratchBuffer);
1937         m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
1938     }
1939     
1940     void compileGetStack()
1941     {
1942         StackAccessData* data = m_node->stackAccessData();
1943         AbstractValue& value = m_state.operand(data->local);
1944         
1945         DFG_ASSERT(m_graph, m_node, isConcrete(data->format), data->format);
1946         
1947         switch (data->format) {
1948         case FlushedDouble:
1949             setDouble(m_out.loadDouble(addressFor(data->machineLocal)));
1950             break;
1951         case FlushedInt52:
1952             setInt52(m_out.load64(addressFor(data->machineLocal)));
1953             break;
1954         default:
1955             if (isInt32Speculation(value.m_type))
1956                 setInt32(m_out.load32(payloadFor(data->machineLocal)));
1957             else
1958                 setJSValue(m_out.load64(addressFor(data->machineLocal)));
1959             break;
1960         }
1961     }
1962     
1963     void compilePutStack()
1964     {
1965         StackAccessData* data = m_node->stackAccessData();
1966         switch (data->format) {
1967         case FlushedJSValue: {
1968             LValue value = lowJSValue(m_node->child1());
1969             m_out.store64(value, addressFor(data->machineLocal));
1970             break;
1971         }
1972             
1973         case FlushedDouble: {
1974             LValue value = lowDouble(m_node->child1());
1975             m_out.storeDouble(value, addressFor(data->machineLocal));
1976             break;
1977         }
1978             
1979         case FlushedInt32: {
1980             LValue value = lowInt32(m_node->child1());
1981             m_out.store32(value, payloadFor(data->machineLocal));
1982             break;
1983         }
1984             
1985         case FlushedInt52: {
1986             LValue value = lowInt52(m_node->child1());
1987             m_out.store64(value, addressFor(data->machineLocal));
1988             break;
1989         }
1990             
1991         case FlushedCell: {
1992             LValue value = lowCell(m_node->child1());
1993             m_out.store64(value, addressFor(data->machineLocal));
1994             break;
1995         }
1996             
1997         case FlushedBoolean: {
1998             speculateBoolean(m_node->child1());
1999             m_out.store64(
2000                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2001                 addressFor(data->machineLocal));
2002             break;
2003         }
2004             
2005         default:
2006             DFG_CRASH(m_graph, m_node, "Bad flush format");
2007             break;
2008         }
2009     }
2010     
2011     void compileNoOp()
2012     {
2013         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
2014     }
2015
2016     void compileToObjectOrCallObjectConstructor()
2017     {
2018         LValue value = lowJSValue(m_node->child1());
2019
2020         LBasicBlock isCellCase = m_out.newBlock();
2021         LBasicBlock slowCase = m_out.newBlock();
2022         LBasicBlock continuation = m_out.newBlock();
2023
2024         m_out.branch(isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2025
2026         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2027         ValueFromBlock fastResult = m_out.anchor(value);
2028         m_out.branch(isObject(value), usually(continuation), rarely(slowCase));
2029
2030         m_out.appendTo(slowCase, continuation);
2031
2032         ValueFromBlock slowResult;
2033         if (m_node->op() == ToObject) {
2034             auto* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2035             slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationToObject), m_callFrame, weakPointer(globalObject), value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2036         } else
2037             slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationCallObjectConstructor), m_callFrame, frozenPointer(m_node->cellOperand()), value));
2038         m_out.jump(continuation);
2039
2040         m_out.appendTo(continuation, lastNext);
2041         setJSValue(m_out.phi(Int64, fastResult, slowResult));
2042     }
2043     
2044     void compileToThis()
2045     {
2046         LValue value = lowJSValue(m_node->child1());
2047         
2048         LBasicBlock isCellCase = m_out.newBlock();
2049         LBasicBlock slowCase = m_out.newBlock();
2050         LBasicBlock continuation = m_out.newBlock();
2051         
2052         m_out.branch(
2053             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2054         
2055         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2056         ValueFromBlock fastResult = m_out.anchor(value);
2057         m_out.branch(
2058             m_out.testIsZero32(
2059                 m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags),
2060                 m_out.constInt32(OverridesToThis)),
2061             usually(continuation), rarely(slowCase));
2062         
2063         m_out.appendTo(slowCase, continuation);
2064         J_JITOperation_EJ function;
2065         if (m_graph.isStrictModeFor(m_node->origin.semantic))
2066             function = operationToThisStrict;
2067         else
2068             function = operationToThis;
2069         ValueFromBlock slowResult = m_out.anchor(
2070             vmCall(Int64, m_out.operation(function), m_callFrame, value));
2071         m_out.jump(continuation);
2072         
2073         m_out.appendTo(continuation, lastNext);
2074         setJSValue(m_out.phi(Int64, fastResult, slowResult));
2075     }
2076
2077     void compileValueAdd()
2078     {
2079         if (m_node->isBinaryUseKind(BigIntUse)) {
2080             LValue left = lowBigInt(m_node->child1());
2081             LValue right = lowBigInt(m_node->child2());
2082
2083             LValue result = vmCall(pointerType(), m_out.operation(operationAddBigInt), m_callFrame, left, right);
2084             setJSValue(result);
2085             return;
2086         }
2087
2088         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2089         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2090         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2091         auto repatchingFunction = operationValueAddOptimize;
2092         auto nonRepatchingFunction = operationValueAdd;
2093         compileBinaryMathIC<JITAddGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2094     }
2095
2096     void compileValueSub()
2097     {
2098         if (m_node->isBinaryUseKind(BigIntUse)) {
2099             LValue left = lowBigInt(m_node->child1());
2100             LValue right = lowBigInt(m_node->child2());
2101             
2102             LValue result = vmCall(pointerType(), m_out.operation(operationSubBigInt), m_callFrame, left, right);
2103             setJSValue(result);
2104             return;
2105         }
2106
2107         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2108         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2109         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2110         auto repatchingFunction = operationValueSubOptimize;
2111         auto nonRepatchingFunction = operationValueSub;
2112         compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2113     }
2114
2115     void compileValueMul()
2116     {
2117         if (m_node->isBinaryUseKind(BigIntUse)) {
2118             LValue left = lowBigInt(m_node->child1());
2119             LValue right = lowBigInt(m_node->child2());
2120             
2121             LValue result = vmCall(Int64, m_out.operation(operationMulBigInt), m_callFrame, left, right);
2122             setJSValue(result);
2123             return;
2124         }
2125
2126         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2127         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2128         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2129         auto repatchingFunction = operationValueMulOptimize;
2130         auto nonRepatchingFunction = operationValueMul;
2131         compileBinaryMathIC<JITMulGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2132     }
2133
2134     template <typename Generator, typename Func1, typename Func2,
2135         typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2136     void compileUnaryMathIC(ArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2137     {
2138         Node* node = m_node;
2139
2140         LValue operand = lowJSValue(node->child1());
2141
2142         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2143         patchpoint->appendSomeRegister(operand);
2144         patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister));
2145         patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister));
2146         RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
2147         patchpoint->numGPScratchRegisters = 1;
2148         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2149         State* state = &m_ftlState;
2150         patchpoint->setGenerator(
2151             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2152                 AllowMacroScratchRegisterUsage allowScratch(jit);
2153
2154                 Box<CCallHelpers::JumpList> exceptions =
2155                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2156
2157 #if ENABLE(MATH_IC_STATS)
2158                 auto inlineStart = jit.label();
2159 #endif
2160
2161                 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2162                 JITUnaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2163                 mathIC->m_generator = Generator(JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), params.gpScratch(0));
2164
2165                 bool shouldEmitProfiling = false;
2166                 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2167
2168                 if (generatedInline) {
2169                     ASSERT(!mathICGenerationState->slowPathJumps.empty());
2170                     auto done = jit.label();
2171                     params.addLatePath([=] (CCallHelpers& jit) {
2172                         AllowMacroScratchRegisterUsage allowScratch(jit);
2173                         mathICGenerationState->slowPathJumps.link(&jit);
2174                         mathICGenerationState->slowPathStart = jit.label();
2175 #if ENABLE(MATH_IC_STATS)
2176                         auto slowPathStart = jit.label();
2177 #endif
2178
2179                         if (mathICGenerationState->shouldSlowPathRepatch) {
2180                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2181                                 repatchingFunction, params[0].gpr(), params[1].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2182                             mathICGenerationState->slowPathCall = call.call();
2183                         } else {
2184                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2185                                 exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr());
2186                             mathICGenerationState->slowPathCall = call.call();
2187                         }
2188                         jit.jump().linkTo(done, &jit);
2189
2190                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2191                             mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2192                         });
2193
2194 #if ENABLE(MATH_IC_STATS)
2195                         auto slowPathEnd = jit.label();
2196                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2197                             size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2198                             mathIC->m_generatedCodeSize += size;
2199                         });
2200 #endif
2201                     });
2202                 } else {
2203                     callOperation(
2204                         *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2205                         nonRepatchingFunction, params[0].gpr(), params[1].gpr());
2206                 }
2207
2208 #if ENABLE(MATH_IC_STATS)
2209                 auto inlineEnd = jit.label();
2210                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2211                     size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2212                     mathIC->m_generatedCodeSize += size;
2213                 });
2214 #endif
2215             });
2216
2217         setJSValue(patchpoint);
2218     }
2219
2220     template <typename Generator, typename Func1, typename Func2,
2221         typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2222     void compileBinaryMathIC(ArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2223     {
2224         Node* node = m_node;
2225         
2226         LValue left = lowJSValue(node->child1());
2227         LValue right = lowJSValue(node->child2());
2228
2229         SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
2230         SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
2231             
2232         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2233         patchpoint->appendSomeRegister(left);
2234         patchpoint->appendSomeRegister(right);
2235         patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister));
2236         patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister));
2237         RefPtr<PatchpointExceptionHandle> exceptionHandle =
2238             preparePatchpointForExceptions(patchpoint);
2239         patchpoint->numGPScratchRegisters = 1;
2240         patchpoint->numFPScratchRegisters = 2;
2241         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2242         State* state = &m_ftlState;
2243         patchpoint->setGenerator(
2244             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2245                 AllowMacroScratchRegisterUsage allowScratch(jit);
2246
2247
2248                 Box<CCallHelpers::JumpList> exceptions =
2249                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2250
2251 #if ENABLE(MATH_IC_STATS)
2252                 auto inlineStart = jit.label();
2253 #endif
2254
2255                 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2256                 JITBinaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2257                 mathIC->m_generator = Generator(leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
2258                     JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0),
2259                     params.fpScratch(1), params.gpScratch(0), InvalidFPRReg);
2260
2261                 bool shouldEmitProfiling = false;
2262                 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2263
2264                 if (generatedInline) {
2265                     ASSERT(!mathICGenerationState->slowPathJumps.empty());
2266                     auto done = jit.label();
2267                     params.addLatePath([=] (CCallHelpers& jit) {
2268                         AllowMacroScratchRegisterUsage allowScratch(jit);
2269                         mathICGenerationState->slowPathJumps.link(&jit);
2270                         mathICGenerationState->slowPathStart = jit.label();
2271 #if ENABLE(MATH_IC_STATS)
2272                         auto slowPathStart = jit.label();
2273 #endif
2274
2275                         if (mathICGenerationState->shouldSlowPathRepatch) {
2276                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2277                                 repatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2278                             mathICGenerationState->slowPathCall = call.call();
2279                         } else {
2280                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2281                                 exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr());
2282                             mathICGenerationState->slowPathCall = call.call();
2283                         }
2284                         jit.jump().linkTo(done, &jit);
2285
2286                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2287                             mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2288                         });
2289
2290 #if ENABLE(MATH_IC_STATS)
2291                         auto slowPathEnd = jit.label();
2292                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2293                             size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2294                             mathIC->m_generatedCodeSize += size;
2295                         });
2296 #endif
2297                     });
2298                 } else {
2299                     callOperation(
2300                         *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2301                         nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr());
2302                 }
2303
2304 #if ENABLE(MATH_IC_STATS)
2305                 auto inlineEnd = jit.label();
2306                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2307                     size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2308                     mathIC->m_generatedCodeSize += size;
2309                 });
2310 #endif
2311             });
2312
2313         setJSValue(patchpoint);
2314     }
2315     
2316     void compileStrCat()
2317     {
2318         LValue result;
2319         if (m_node->child3()) {
2320             result = vmCall(
2321                 Int64, m_out.operation(operationStrCat3), m_callFrame,
2322                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2323                 lowJSValue(m_node->child2(), ManualOperandSpeculation),
2324                 lowJSValue(m_node->child3(), ManualOperandSpeculation));
2325         } else {
2326             result = vmCall(
2327                 Int64, m_out.operation(operationStrCat2), m_callFrame,
2328                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2329                 lowJSValue(m_node->child2(), ManualOperandSpeculation));
2330         }
2331         setJSValue(result);
2332     }
2333     
2334     void compileArithAddOrSub()
2335     {
2336         bool isSub =  m_node->op() == ArithSub;
2337         switch (m_node->binaryUseKind()) {
2338         case Int32Use: {
2339             LValue left = lowInt32(m_node->child1());
2340             LValue right = lowInt32(m_node->child2());
2341
2342             if (!shouldCheckOverflow(m_node->arithMode())) {
2343                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
2344                 break;
2345             }
2346
2347             CheckValue* result =
2348                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2349             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2350             setInt32(result);
2351             break;
2352         }
2353             
2354         case Int52RepUse: {
2355             if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)
2356                 && !abstractValue(m_node->child2()).couldBeType(SpecNonInt32AsInt52)) {
2357                 Int52Kind kind;
2358                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
2359                 LValue right = lowInt52(m_node->child2(), kind);
2360                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
2361                 break;
2362             }
2363
2364             LValue left = lowInt52(m_node->child1());
2365             LValue right = lowInt52(m_node->child2());
2366             CheckValue* result =
2367                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2368             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2369             setInt52(result);
2370             break;
2371         }
2372             
2373         case DoubleRepUse: {
2374             LValue C1 = lowDouble(m_node->child1());
2375             LValue C2 = lowDouble(m_node->child2());
2376
2377             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
2378             break;
2379         }
2380
2381         case UntypedUse: {
2382             if (!isSub) {
2383                 DFG_CRASH(m_graph, m_node, "Bad use kind");
2384                 break;
2385             }
2386
2387             CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2388             unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2389             ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2390             auto repatchingFunction = operationValueSubOptimize;
2391             auto nonRepatchingFunction = operationValueSub;
2392             compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2393             break;
2394         }
2395
2396         default:
2397             DFG_CRASH(m_graph, m_node, "Bad use kind");
2398             break;
2399         }
2400     }
2401
2402     void compileArithClz32()
2403     {
2404         if (m_node->child1().useKind() == Int32Use || m_node->child1().useKind() == KnownInt32Use) {
2405             LValue operand = lowInt32(m_node->child1());
2406             setInt32(m_out.ctlz32(operand));
2407             return;
2408         }
2409         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2410         LValue argument = lowJSValue(m_node->child1());
2411         LValue result = vmCall(Int32, m_out.operation(operationArithClz32), m_callFrame, argument);
2412         setInt32(result);
2413     }
2414     
2415     void compileArithMul()
2416     {
2417         switch (m_node->binaryUseKind()) {
2418         case Int32Use: {
2419             LValue left = lowInt32(m_node->child1());
2420             LValue right = lowInt32(m_node->child2());
2421             
2422             LValue result;
2423
2424             if (!shouldCheckOverflow(m_node->arithMode()))
2425                 result = m_out.mul(left, right);
2426             else {
2427                 CheckValue* speculation = m_out.speculateMul(left, right);
2428                 blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin);
2429                 result = speculation;
2430             }
2431             
2432             if (shouldCheckNegativeZero(m_node->arithMode())) {
2433                 LBasicBlock slowCase = m_out.newBlock();
2434                 LBasicBlock continuation = m_out.newBlock();
2435                 
2436                 m_out.branch(
2437                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
2438                 
2439                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2440                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero));
2441                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero));
2442                 m_out.jump(continuation);
2443                 m_out.appendTo(continuation, lastNext);
2444             }
2445             
2446             setInt32(result);
2447             break;
2448         }
2449             
2450         case Int52RepUse: {
2451             Int52Kind kind;
2452             LValue left = lowWhicheverInt52(m_node->child1(), kind);
2453             LValue right = lowInt52(m_node->child2(), opposite(kind));
2454
2455             CheckValue* result = m_out.speculateMul(left, right);
2456             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2457
2458             if (shouldCheckNegativeZero(m_node->arithMode())) {
2459                 LBasicBlock slowCase = m_out.newBlock();
2460                 LBasicBlock continuation = m_out.newBlock();
2461                 
2462                 m_out.branch(
2463                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2464                 
2465                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2466                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero));
2467                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero));
2468                 m_out.jump(continuation);
2469                 m_out.appendTo(continuation, lastNext);
2470             }
2471             
2472             setInt52(result);
2473             break;
2474         }
2475             
2476         case DoubleRepUse: {
2477             setDouble(
2478                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2479             break;
2480         }
2481
2482         default:
2483             DFG_CRASH(m_graph, m_node, "Bad use kind");
2484             break;
2485         }
2486     }
2487
2488     void compileValueDiv()
2489     {
2490         if (m_node->isBinaryUseKind(BigIntUse)) {
2491             LValue left = lowBigInt(m_node->child1());
2492             LValue right = lowBigInt(m_node->child2());
2493             
2494             LValue result = vmCall(pointerType(), m_out.operation(operationDivBigInt), m_callFrame, left, right);
2495             setJSValue(result);
2496             return;
2497         }
2498
2499         emitBinarySnippet<JITDivGenerator, NeedScratchFPR>(operationValueDiv);
2500     }
2501
2502     void compileArithDiv()
2503     {
2504         switch (m_node->binaryUseKind()) {
2505         case Int32Use: {
2506             LValue numerator = lowInt32(m_node->child1());
2507             LValue denominator = lowInt32(m_node->child2());
2508
2509             if (shouldCheckNegativeZero(m_node->arithMode())) {
2510                 LBasicBlock zeroNumerator = m_out.newBlock();
2511                 LBasicBlock numeratorContinuation = m_out.newBlock();
2512
2513                 m_out.branch(
2514                     m_out.isZero32(numerator),
2515                     rarely(zeroNumerator), usually(numeratorContinuation));
2516
2517                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
2518
2519                 speculate(
2520                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
2521
2522                 m_out.jump(numeratorContinuation);
2523
2524                 m_out.appendTo(numeratorContinuation, innerLastNext);
2525             }
2526             
2527             if (shouldCheckOverflow(m_node->arithMode())) {
2528                 LBasicBlock unsafeDenominator = m_out.newBlock();
2529                 LBasicBlock continuation = m_out.newBlock();
2530
2531                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2532                 m_out.branch(
2533                     m_out.above(adjustedDenominator, m_out.int32One),
2534                     usually(continuation), rarely(unsafeDenominator));
2535
2536                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2537                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2538                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2539                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2540                 m_out.jump(continuation);
2541
2542                 m_out.appendTo(continuation, lastNext);
2543                 LValue result = m_out.div(numerator, denominator);
2544                 speculate(
2545                     Overflow, noValue(), 0,
2546                     m_out.notEqual(m_out.mul(result, denominator), numerator));
2547                 setInt32(result);
2548             } else
2549                 setInt32(m_out.chillDiv(numerator, denominator));
2550
2551             break;
2552         }
2553             
2554         case DoubleRepUse: {
2555             setDouble(m_out.doubleDiv(
2556                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2557             break;
2558         }
2559
2560         default:
2561             DFG_CRASH(m_graph, m_node, "Bad use kind");
2562             break;
2563         }
2564     }
2565     
2566     void compileValueMod()
2567     {
2568         if (m_node->binaryUseKind() == BigIntUse) {
2569             LValue left = lowBigInt(m_node->child1());
2570             LValue right = lowBigInt(m_node->child2());
2571
2572             LValue result = vmCall(pointerType(), m_out.operation(operationModBigInt), m_callFrame, left, right);
2573             setJSValue(result);
2574             return;
2575         }
2576
2577         DFG_ASSERT(m_graph, m_node, m_node->binaryUseKind() == UntypedUse, m_node->binaryUseKind());
2578         LValue left = lowJSValue(m_node->child1());
2579         LValue right = lowJSValue(m_node->child2());
2580         LValue result = vmCall(Int64, m_out.operation(operationValueMod), m_callFrame, left, right);
2581         setJSValue(result);
2582     }
2583
2584     void compileArithMod()
2585     {
2586         switch (m_node->binaryUseKind()) {
2587         case Int32Use: {
2588             LValue numerator = lowInt32(m_node->child1());
2589             LValue denominator = lowInt32(m_node->child2());
2590
2591             LValue remainder;
2592             if (shouldCheckOverflow(m_node->arithMode())) {
2593                 LBasicBlock unsafeDenominator = m_out.newBlock();
2594                 LBasicBlock continuation = m_out.newBlock();
2595
2596                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2597                 m_out.branch(
2598                     m_out.above(adjustedDenominator, m_out.int32One),
2599                     usually(continuation), rarely(unsafeDenominator));
2600
2601                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2602                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2603                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2604                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2605                 m_out.jump(continuation);
2606
2607                 m_out.appendTo(continuation, lastNext);
2608                 LValue result = m_out.mod(numerator, denominator);
2609                 remainder = result;
2610             } else
2611                 remainder = m_out.chillMod(numerator, denominator);
2612
2613             if (shouldCheckNegativeZero(m_node->arithMode())) {
2614                 LBasicBlock negativeNumerator = m_out.newBlock();
2615                 LBasicBlock numeratorContinuation = m_out.newBlock();
2616
2617                 m_out.branch(
2618                     m_out.lessThan(numerator, m_out.int32Zero),
2619                     unsure(negativeNumerator), unsure(numeratorContinuation));
2620
2621                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
2622
2623                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
2624
2625                 m_out.jump(numeratorContinuation);
2626
2627                 m_out.appendTo(numeratorContinuation, innerLastNext);
2628             }
2629
2630             setInt32(remainder);
2631             break;
2632         }
2633             
2634         case DoubleRepUse: {
2635             setDouble(
2636                 m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2637             break;
2638         }
2639             
2640         default:
2641             DFG_CRASH(m_graph, m_node, "Bad use kind");
2642             break;
2643         }
2644     }
2645
2646     void compileArithMinOrMax()
2647     {
2648         switch (m_node->binaryUseKind()) {
2649         case Int32Use: {
2650             LValue left = lowInt32(m_node->child1());
2651             LValue right = lowInt32(m_node->child2());
2652             
2653             setInt32(
2654                 m_out.select(
2655                     m_node->op() == ArithMin
2656                         ? m_out.lessThan(left, right)
2657                         : m_out.lessThan(right, left),
2658                     left, right));
2659             break;
2660         }
2661             
2662         case DoubleRepUse: {
2663             LValue left = lowDouble(m_node->child1());
2664             LValue right = lowDouble(m_node->child2());
2665             
2666             LBasicBlock notLessThan = m_out.newBlock();
2667             LBasicBlock continuation = m_out.newBlock();
2668             
2669             Vector<ValueFromBlock, 2> results;
2670             
2671             results.append(m_out.anchor(left));
2672             m_out.branch(
2673                 m_node->op() == ArithMin
2674                     ? m_out.doubleLessThan(left, right)
2675                     : m_out.doubleGreaterThan(left, right),
2676                 unsure(continuation), unsure(notLessThan));
2677             
2678             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
2679             results.append(m_out.anchor(m_out.select(
2680                 m_node->op() == ArithMin
2681                     ? m_out.doubleGreaterThanOrEqual(left, right)
2682                     : m_out.doubleLessThanOrEqual(left, right),
2683                 right, m_out.constDouble(PNaN))));
2684             m_out.jump(continuation);
2685             
2686             m_out.appendTo(continuation, lastNext);
2687             setDouble(m_out.phi(Double, results));
2688             break;
2689         }
2690             
2691         default:
2692             DFG_CRASH(m_graph, m_node, "Bad use kind");
2693             break;
2694         }
2695     }
2696     
2697     void compileArithAbs()
2698     {
2699         switch (m_node->child1().useKind()) {
2700         case Int32Use: {
2701             LValue value = lowInt32(m_node->child1());
2702
2703             LValue mask = m_out.aShr(value, m_out.constInt32(31));
2704             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
2705
2706             if (shouldCheckOverflow(m_node->arithMode()))
2707                 speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2708
2709             setInt32(result);
2710             break;
2711         }
2712             
2713         case DoubleRepUse: {
2714             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
2715             break;
2716         }
2717             
2718         default: {
2719             DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2720             LValue argument = lowJSValue(m_node->child1());
2721             LValue result = vmCall(Double, m_out.operation(operationArithAbs), m_callFrame, argument);
2722             setDouble(result);
2723             break;
2724         }
2725         }
2726     }
2727
2728     void compileArithUnary()
2729     {
2730         if (m_node->child1().useKind() == DoubleRepUse) {
2731             setDouble(m_out.doubleUnary(m_node->arithUnaryType(), lowDouble(m_node->child1())));
2732             return;
2733         }
2734         LValue argument = lowJSValue(m_node->child1());
2735         LValue result = vmCall(Double, m_out.operation(DFG::arithUnaryOperation(m_node->arithUnaryType())), m_callFrame, argument);
2736         setDouble(result);
2737     }
2738
2739     void compileValuePow()
2740     {
2741         if (m_node->isBinaryUseKind(BigIntUse)) {
2742             LValue base = lowBigInt(m_node->child1());
2743             LValue exponent = lowBigInt(m_node->child2());
2744             
2745             LValue result = vmCall(pointerType(), m_out.operation(operationPowBigInt), m_callFrame, base, exponent);
2746             setJSValue(result);
2747             return;
2748         }
2749
2750         LValue base = lowJSValue(m_node->child1());
2751         LValue exponent = lowJSValue(m_node->child2());
2752         LValue result = vmCall(Int64, m_out.operation(operationValuePow), m_callFrame, base, exponent);
2753         setJSValue(result);
2754     }
2755
2756     void compileArithPow()
2757     {
2758         if (m_node->child2().useKind() == Int32Use)
2759             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
2760         else {
2761             LValue base = lowDouble(m_node->child1());
2762             LValue exponent = lowDouble(m_node->child2());
2763
2764             LBasicBlock integerExponentIsSmallBlock = m_out.newBlock();
2765             LBasicBlock integerExponentPowBlock = m_out.newBlock();
2766             LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock();
2767             LBasicBlock nanExceptionBaseIsOne = m_out.newBlock();
2768             LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock();
2769             LBasicBlock testExponentIsOneHalf = m_out.newBlock();
2770             LBasicBlock handleBaseZeroExponentIsOneHalf = m_out.newBlock();
2771             LBasicBlock handleInfinityForExponentIsOneHalf = m_out.newBlock();
2772             LBasicBlock exponentIsOneHalfNormal = m_out.newBlock();
2773             LBasicBlock exponentIsOneHalfInfinity = m_out.newBlock();
2774             LBasicBlock testExponentIsNegativeOneHalf = m_out.newBlock();
2775             LBasicBlock testBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2776             LBasicBlock handleBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2777             LBasicBlock handleInfinityForExponentIsNegativeOneHalf = m_out.newBlock();
2778             LBasicBlock exponentIsNegativeOneHalfNormal = m_out.newBlock();
2779             LBasicBlock exponentIsNegativeOneHalfInfinity = m_out.newBlock();
2780             LBasicBlock powBlock = m_out.newBlock();
2781             LBasicBlock nanExceptionResultIsNaN = m_out.newBlock();
2782             LBasicBlock continuation = m_out.newBlock();
2783
2784             LValue integerExponent = m_out.doubleToInt(exponent);
2785             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
2786             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
2787             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
2788
2789             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
2790             LValue integerExponentBelowMax = m_out.belowOrEqual(integerExponent, m_out.constInt32(maxExponentForIntegerMathPow));
2791             m_out.branch(integerExponentBelowMax, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
2792
2793             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
2794             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
2795             m_out.jump(continuation);
2796
2797             // If y is NaN, the result is NaN.
2798             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionBaseIsOne);
2799             LValue exponentIsNaN;
2800             if (provenType(m_node->child2()) & SpecDoubleNaN)
2801                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
2802             else
2803                 exponentIsNaN = m_out.booleanFalse;
2804             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionBaseIsOne));
2805
2806             // If abs(x) is 1 and y is +infinity, the result is NaN.
2807             // If abs(x) is 1 and y is -infinity, the result is NaN.
2808
2809             //     Test if base == 1.
2810             m_out.appendTo(nanExceptionBaseIsOne, nanExceptionExponentIsInfinity);
2811             LValue absoluteBase = m_out.doubleAbs(base);
2812             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
2813             m_out.branch(absoluteBaseIsOne, rarely(nanExceptionExponentIsInfinity), usually(testExponentIsOneHalf));
2814
2815             //     Test if abs(y) == Infinity.
2816             m_out.appendTo(nanExceptionExponentIsInfinity, testExponentIsOneHalf);
2817             LValue absoluteExponent = m_out.doubleAbs(exponent);
2818             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
2819             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionResultIsNaN), usually(testExponentIsOneHalf));
2820
2821             // If y == 0.5 or y == -0.5, handle it through SQRT.
2822             // We have be carefuly with -0 and -Infinity.
2823
2824             //     Test if y == 0.5
2825             m_out.appendTo(testExponentIsOneHalf, handleBaseZeroExponentIsOneHalf);
2826             LValue exponentIsOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(0.5));
2827             m_out.branch(exponentIsOneHalf, rarely(handleBaseZeroExponentIsOneHalf), usually(testExponentIsNegativeOneHalf));
2828
2829             //     Handle x == -0.
2830             m_out.appendTo(handleBaseZeroExponentIsOneHalf, handleInfinityForExponentIsOneHalf);
2831             LValue baseIsZeroExponentIsOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2832             ValueFromBlock zeroResultExponentIsOneHalf = m_out.anchor(m_out.doubleZero);
2833             m_out.branch(baseIsZeroExponentIsOneHalf, rarely(continuation), usually(handleInfinityForExponentIsOneHalf));
2834
2835             //     Test if abs(x) == Infinity.
2836             m_out.appendTo(handleInfinityForExponentIsOneHalf, exponentIsOneHalfNormal);
2837             LValue absoluteBaseIsInfinityOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2838             m_out.branch(absoluteBaseIsInfinityOneHalf, rarely(exponentIsOneHalfInfinity), usually(exponentIsOneHalfNormal));
2839
2840             //     The exponent is 0.5, the base is finite or NaN, we can use SQRT.
2841             m_out.appendTo(exponentIsOneHalfNormal, exponentIsOneHalfInfinity);
2842             ValueFromBlock sqrtResult = m_out.anchor(m_out.doubleSqrt(base));
2843             m_out.jump(continuation);
2844
2845             //     The exponent is 0.5, the base is infinite, the result is always infinite.
2846             m_out.appendTo(exponentIsOneHalfInfinity, testExponentIsNegativeOneHalf);
2847             ValueFromBlock sqrtInfinityResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2848             m_out.jump(continuation);
2849
2850             //     Test if y == -0.5
2851             m_out.appendTo(testExponentIsNegativeOneHalf, testBaseZeroExponentIsNegativeOneHalf);
2852             LValue exponentIsNegativeOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(-0.5));
2853             m_out.branch(exponentIsNegativeOneHalf, rarely(testBaseZeroExponentIsNegativeOneHalf), usually(powBlock));
2854
2855             //     Handle x == -0.
2856             m_out.appendTo(testBaseZeroExponentIsNegativeOneHalf, handleBaseZeroExponentIsNegativeOneHalf);
2857             LValue baseIsZeroExponentIsNegativeOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2858             m_out.branch(baseIsZeroExponentIsNegativeOneHalf, rarely(handleBaseZeroExponentIsNegativeOneHalf), usually(handleInfinityForExponentIsNegativeOneHalf));
2859
2860             m_out.appendTo(handleBaseZeroExponentIsNegativeOneHalf, handleInfinityForExponentIsNegativeOneHalf);
2861             ValueFromBlock oneOverSqrtZeroResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2862             m_out.jump(continuation);
2863
2864             //     Test if abs(x) == Infinity.
2865             m_out.appendTo(handleInfinityForExponentIsNegativeOneHalf, exponentIsNegativeOneHalfNormal);
2866             LValue absoluteBaseIsInfinityNegativeOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2867             m_out.branch(absoluteBaseIsInfinityNegativeOneHalf, rarely(exponentIsNegativeOneHalfInfinity), usually(exponentIsNegativeOneHalfNormal));
2868
2869             //     The exponent is -0.5, the base is finite or NaN, we can use 1/SQRT.
2870             m_out.appendTo(exponentIsNegativeOneHalfNormal, exponentIsNegativeOneHalfInfinity);
2871             LValue sqrtBase = m_out.doubleSqrt(base);
2872             ValueFromBlock oneOverSqrtResult = m_out.anchor(m_out.div(m_out.constDouble(1.), sqrtBase));
2873             m_out.jump(continuation);
2874
2875             //     The exponent is -0.5, the base is infinite, the result is always zero.
2876             m_out.appendTo(exponentIsNegativeOneHalfInfinity, powBlock);
2877             ValueFromBlock oneOverSqrtInfinityResult = m_out.anchor(m_out.doubleZero);
2878             m_out.jump(continuation);
2879
2880             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
2881             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
2882             m_out.jump(continuation);
2883
2884             m_out.appendTo(nanExceptionResultIsNaN, continuation);
2885             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
2886             m_out.jump(continuation);
2887
2888             m_out.appendTo(continuation, lastNext);
2889             setDouble(m_out.phi(Double, powDoubleIntResult, zeroResultExponentIsOneHalf, sqrtResult, sqrtInfinityResult, oneOverSqrtZeroResult, oneOverSqrtResult, oneOverSqrtInfinityResult, powResult, pureNan));
2890         }
2891     }
2892
2893     void compileArithRandom()
2894     {
2895         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2896
2897         // Inlined WeakRandom::advance().
2898         // uint64_t x = m_low;
2899         void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
2900         LValue low = m_out.load64(m_out.absolute(lowAddress));
2901         // uint64_t y = m_high;
2902         void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();
2903         LValue high = m_out.load64(m_out.absolute(highAddress));
2904         // m_low = y;
2905         m_out.store64(high, m_out.absolute(lowAddress));
2906
2907         // x ^= x << 23;
2908         LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low);
2909
2910         // x ^= x >> 17;
2911         LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1);
2912
2913         // x ^= y ^ (y >> 26);
2914         LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2);
2915
2916         // m_high = x;
2917         m_out.store64(phase3, m_out.absolute(highAddress));
2918
2919         // return x + y;
2920         LValue random64 = m_out.add(phase3, high);
2921
2922         // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
2923         LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1));
2924
2925         LValue double53Integer = m_out.intToDouble(random53);
2926
2927         // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
2928         // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
2929         static const double scale = 1.0 / (1ULL << 53);
2930
2931         // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
2932         // It just reduces the exp part of the given 53bit double integer.
2933         // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
2934         // Now we get 53bit precision random double value in [0, 1).
2935         LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale));
2936
2937         setDouble(result);
2938     }
2939
2940     void compileArithRound()
2941     {
2942         if (m_node->child1().useKind() == DoubleRepUse) {
2943             LValue result = nullptr;
2944             if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) {
2945                 LValue value = lowDouble(m_node->child1());
2946                 result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5)));
2947             } else {
2948                 LBasicBlock realPartIsMoreThanHalf = m_out.newBlock();
2949                 LBasicBlock continuation = m_out.newBlock();
2950
2951                 LValue value = lowDouble(m_node->child1());
2952                 LValue integerValue = m_out.doubleCeil(value);
2953                 ValueFromBlock integerValueResult = m_out.anchor(integerValue);
2954
2955                 LValue realPart = m_out.doubleSub(integerValue, value);
2956
2957                 m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
2958
2959                 LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
2960                 LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
2961                 ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
2962                 m_out.jump(continuation);
2963                 m_out.appendTo(continuation, lastNext);
2964
2965                 result = m_out.phi(Double, integerValueResult, integerValueRoundedDownResult);
2966             }
2967
2968             if (producesInteger(m_node->arithRoundingMode())) {
2969                 LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
2970                 setInt32(integerValue);
2971             } else
2972                 setDouble(result);
2973             return;
2974         }
2975
2976         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2977         LValue argument = lowJSValue(m_node->child1());
2978         setJSValue(vmCall(Int64, m_out.operation(operationArithRound), m_callFrame, argument));
2979     }
2980
2981     void compileArithFloor()
2982     {
2983         if (m_node->child1().useKind() == DoubleRepUse) {
2984             LValue value = lowDouble(m_node->child1());
2985             LValue integerValue = m_out.doubleFloor(value);
2986             if (producesInteger(m_node->arithRoundingMode()))
2987                 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2988             else
2989                 setDouble(integerValue);
2990             return;
2991         }
2992         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2993         LValue argument = lowJSValue(m_node->child1());
2994         setJSValue(vmCall(Int64, m_out.operation(operationArithFloor), m_callFrame, argument));
2995     }
2996
2997     void compileArithCeil()
2998     {
2999         if (m_node->child1().useKind() == DoubleRepUse) {
3000             LValue value = lowDouble(m_node->child1());
3001             LValue integerValue = m_out.doubleCeil(value);
3002             if (producesInteger(m_node->arithRoundingMode()))
3003                 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
3004             else
3005                 setDouble(integerValue);
3006             return;
3007         }
3008         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3009         LValue argument = lowJSValue(m_node->child1());
3010         setJSValue(vmCall(Int64, m_out.operation(operationArithCeil), m_callFrame, argument));
3011     }
3012
3013     void compileArithTrunc()
3014     {
3015         if (m_node->child1().useKind() == DoubleRepUse) {
3016             LValue value = lowDouble(m_node->child1());
3017             LValue result = m_out.doubleTrunc(value);
3018             if (producesInteger(m_node->arithRoundingMode()))
3019                 setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())));
3020             else
3021                 setDouble(result);
3022             return;
3023         }
3024         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3025         LValue argument = lowJSValue(m_node->child1());
3026         setJSValue(vmCall(Int64, m_out.operation(operationArithTrunc), m_callFrame, argument));
3027     }
3028
3029     void compileArithSqrt()
3030     {
3031         if (m_node->child1().useKind() == DoubleRepUse) {
3032             setDouble(m_out.doubleSqrt(lowDouble(m_node->child1())));
3033             return;
3034         }
3035         LValue argument = lowJSValue(m_node->child1());
3036         LValue result = vmCall(Double, m_out.operation(operationArithSqrt), m_callFrame, argument);
3037         setDouble(result);
3038     }
3039
3040     void compileArithFRound()
3041     {
3042         if (m_node->child1().useKind() == DoubleRepUse) {
3043             setDouble(m_out.fround(lowDouble(m_node->child1())));
3044             return;
3045         }
3046         LValue argument = lowJSValue(m_node->child1());
3047         LValue result = vmCall(Double, m_out.operation(operationArithFRound), m_callFrame, argument);
3048         setDouble(result);
3049     }
3050
3051     void compileValueNegate()
3052     {
3053         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse);
3054         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
3055         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
3056         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
3057         auto repatchingFunction = operationArithNegateOptimize;
3058         auto nonRepatchingFunction = operationArithNegate;
3059         compileUnaryMathIC<JITNegGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
3060     }
3061
3062     void compileArithNegate()
3063     {
3064         switch (m_node->child1().useKind()) {
3065         case Int32Use: {
3066             LValue value = lowInt32(m_node->child1());
3067             
3068             LValue result;
3069             if (!shouldCheckOverflow(m_node->arithMode()))
3070                 result = m_out.neg(value);
3071             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
3072                 CheckValue* check = m_out.speculateSub(m_out.int32Zero, value);
3073                 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
3074                 result = check;
3075             } else {
3076                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
3077                 result = m_out.neg(value);
3078             }
3079
3080             setInt32(result);
3081             break;
3082         }
3083             
3084         case Int52RepUse: {
3085             if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)) {
3086                 Int52Kind kind;
3087                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
3088                 LValue result = m_out.neg(value);
3089                 if (shouldCheckNegativeZero(m_node->arithMode()))
3090                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3091                 setInt52(result, kind);
3092                 break;
3093             }
3094             
3095             LValue value = lowInt52(m_node->child1());
3096             CheckValue* result = m_out.speculateSub(m_out.int64Zero, value);
3097             blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin);
3098             if (shouldCheckNegativeZero(m_node->arithMode()))
3099                 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3100             setInt52(result);
3101             break;
3102         }
3103             
3104         case DoubleRepUse: {
3105             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
3106             break;
3107         }
3108             
3109         default:
3110             DFG_CRASH(m_graph, m_node, "Bad use kind");
3111             break;
3112         }
3113     }
3114     
3115     void compileValueBitNot()
3116     {
3117         if (m_node->child1().useKind() == BigIntUse) {
3118             LValue operand = lowBigInt(m_node->child1());
3119             LValue result = vmCall(pointerType(), m_out.operation(operationBitNotBigInt), m_callFrame, operand);
3120             setJSValue(result);
3121             return;
3122         }
3123
3124         LValue operand = lowJSValue(m_node->child1());
3125         LValue result = vmCall(Int64, m_out.operation(operationValueBitNot), m_callFrame, operand);
3126         setJSValue(result);
3127     }
3128
3129     void compileArithBitNot()
3130     {
3131         setInt32(m_out.bitNot(lowInt32(m_node->child1())));
3132     }
3133
3134     void compileValueBitAnd()
3135     {
3136         if (m_node->isBinaryUseKind(BigIntUse)) {
3137             LValue left = lowBigInt(m_node->child1());
3138             LValue right = lowBigInt(m_node->child2());
3139             
3140             LValue result = vmCall(pointerType(), m_out.operation(operationBitAndBigInt), m_callFrame, left, right);
3141             setJSValue(result);
3142             return;
3143         }
3144         
3145         emitBinaryBitOpSnippet<JITBitAndGenerator>(operationValueBitAnd);
3146     }
3147     
3148     void compileArithBitAnd()
3149     {
3150         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3151     }
3152     
3153     void compileValueBitOr()
3154     {
3155         if (m_node->isBinaryUseKind(BigIntUse)) {
3156             LValue left = lowBigInt(m_node->child1());
3157             LValue right = lowBigInt(m_node->child2());
3158
3159             LValue result = vmCall(pointerType(), m_out.operation(operationBitOrBigInt), m_callFrame, left, right);
3160             setJSValue(result);
3161             return;
3162         }
3163
3164         emitBinaryBitOpSnippet<JITBitOrGenerator>(operationValueBitOr);
3165     }
3166
3167     void compileArithBitOr()
3168     {
3169         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3170     }
3171     
3172     void compileValueBitXor()
3173     {
3174         if (m_node->isBinaryUseKind(BigIntUse)) {
3175             LValue left = lowBigInt(m_node->child1());
3176             LValue right = lowBigInt(m_node->child2());
3177
3178             LValue result = vmCall(pointerType(), m_out.operation(operationBitXorBigInt), m_callFrame, left, right);
3179             setJSValue(result);
3180             return;
3181         }
3182
3183         emitBinaryBitOpSnippet<JITBitXorGenerator>(operationValueBitXor);
3184     }
3185
3186     void compileArithBitXor()
3187     {
3188         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3189     }
3190     
3191     void compileBitRShift()
3192     {
3193         if (m_node->isBinaryUseKind(UntypedUse)) {
3194             emitRightShiftSnippet(JITRightShiftGenerator::SignedShift);
3195             return;
3196         }
3197         setInt32(m_out.aShr(
3198             lowInt32(m_node->child1()),
3199             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3200     }
3201     
3202     void compileArithBitLShift()
3203     {
3204         setInt32(m_out.shl(
3205             lowInt32(m_node->child1()),
3206             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3207     }
3208     
3209     void compileValueBitLShift()
3210     {
3211         if (m_node->isBinaryUseKind(BigIntUse)) {
3212             LValue left = lowBigInt(m_node->child1());
3213             LValue right = lowBigInt(m_node->child2());
3214             
3215             LValue result = vmCall(pointerType(), m_out.operation(operationBitLShiftBigInt), m_callFrame, left, right);
3216             setJSValue(result);
3217             return;
3218         }
3219
3220         ASSERT(m_node->isBinaryUseKind(UntypedUse));
3221         emitBinaryBitOpSnippet<JITLeftShiftGenerator>(operationValueBitLShift);
3222     }
3223
3224     void compileBitURShift()
3225     {
3226         if (m_node->isBinaryUseKind(UntypedUse)) {
3227             emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift);
3228             return;
3229         }
3230         setInt32(m_out.lShr(
3231             lowInt32(m_node->child1()),
3232             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3233     }
3234     
3235     void compileUInt32ToNumber()
3236     {
3237         LValue value = lowInt32(m_node->child1());
3238
3239         if (doesOverflow(m_node->arithMode())) {
3240             setStrictInt52(m_out.zeroExtPtr(value));
3241             return;
3242         }
3243
3244         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
3245         setInt32(value);
3246     }
3247     
3248     void compileCheckStructure()
3249     {
3250         ExitKind exitKind;
3251         if (m_node->child1()->hasConstant())
3252             exitKind = BadConstantCache;
3253         else
3254             exitKind = BadCache;
3255
3256         switch (m_node->child1().useKind()) {
3257         case CellUse:
3258         case KnownCellUse: {
3259             LValue cell = lowCell(m_node->child1());
3260             
3261             checkStructure(
3262                 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3263                 exitKind, m_node->structureSet(),
3264                 [&] (RegisteredStructure structure) {
3265                     return weakStructureID(structure);
3266                 });
3267             return;
3268         }
3269
3270         case CellOrOtherUse: {
3271             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
3272
3273             LBasicBlock cellCase = m_out.newBlock();
3274             LBasicBlock notCellCase = m_out.newBlock();
3275             LBasicBlock continuation = m_out.newBlock();
3276
3277             m_out.branch(
3278                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3279
3280             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3281             checkStructure(
3282                 m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value),
3283                 exitKind, m_node->structureSet(),
3284                 [&] (RegisteredStructure structure) {
3285                     return weakStructureID(structure);
3286                 });
3287             m_out.jump(continuation);
3288
3289             m_out.appendTo(notCellCase, continuation);
3290             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value));
3291             m_out.jump(continuation);
3292
3293             m_out.appendTo(continuation, lastNext);
3294             return;
3295         }
3296
3297         default:
3298             DFG_CRASH(m_graph, m_node, "Bad use kind");
3299             return;
3300         }
3301     }
3302
3303     void compileCheckStructureOrEmpty()
3304     {
3305         ExitKind exitKind;
3306         if (m_node->child1()->hasConstant())
3307             exitKind = BadConstantCache;
3308         else
3309             exitKind = BadCache;
3310
3311         LValue cell = lowCell(m_node->child1());
3312         bool maySeeEmptyValue = m_interpreter.forNode(m_node->child1()).m_type & SpecEmpty;
3313         LBasicBlock notEmpty;
3314         LBasicBlock continuation;
3315         LBasicBlock lastNext;
3316         if (maySeeEmptyValue) {
3317             notEmpty = m_out.newBlock();
3318             continuation = m_out.newBlock();
3319             m_out.branch(m_out.isZero64(cell), unsure(continuation), unsure(notEmpty));
3320             lastNext = m_out.appendTo(notEmpty, continuation);
3321         }
3322
3323         checkStructure(
3324             m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3325             exitKind, m_node->structureSet(),
3326             [&] (RegisteredStructure structure) {
3327                 return weakStructureID(structure);
3328             });
3329
3330         if (maySeeEmptyValue) {
3331             m_out.jump(continuation);
3332             m_out.appendTo(continuation, lastNext);
3333         }
3334     }
3335     
3336     void compileCheckCell()
3337     {
3338         LValue cell = lowCell(m_node->child1());
3339         
3340         speculate(
3341             BadCell, jsValueValue(cell), m_node->child1().node(),
3342             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
3343     }
3344     
3345     void compileCheckBadCell()
3346     {
3347         terminate(BadCell);
3348     }
3349
3350     void compileCheckNotEmpty()
3351     {
3352         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
3353     }
3354
3355     void compileAssertNotEmpty()
3356     {
3357         if (!validationEnabled())
3358             return;
3359
3360         LValue val = lowJSValue(m_node->child1());
3361         PatchpointValue* patchpoint = m_out.patchpoint(Void);
3362         patchpoint->appendSomeRegister(val);
3363         patchpoint->setGenerator(
3364             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
3365                 AllowMacroScratchRegisterUsage allowScratch(jit);
3366                 GPRReg input =  params[0].gpr();
3367                 CCallHelpers::Jump done = jit.branchIfNotEmpty(input);
3368                 jit.breakpoint();
3369                 done.link(&jit);
3370             });
3371     }
3372
3373     void compileCheckStringIdent()
3374     {
3375         UniquedStringImpl* uid = m_node->uidOperand();
3376         LValue stringImpl = lowStringIdent(m_node->child1());
3377         speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
3378     }
3379
3380     void compileGetExecutable()
3381     {
3382         LValue cell = lowCell(m_node->child1());
3383         speculateFunction(m_node->child1(), cell);
3384         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
3385     }
3386     
3387     void compileArrayify()
3388     {
3389         LValue cell = lowCell(m_node->child1());
3390         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
3391         
3392         LBasicBlock unexpectedStructure = m_out.newBlock();
3393         LBasicBlock continuation = m_out.newBlock();
3394         
3395         auto isUnexpectedArray = [&] (LValue cell) {
3396             if (m_node->op() == Arrayify)
3397                 return m_out.logicalNot(isArrayTypeForArrayify(cell, m_node->arrayMode()));
3398
3399             ASSERT(m_node->op() == ArrayifyToStructure);
3400             return m_out.notEqual(m_out.load32(cell, m_heaps.JSCell_structureID), weakStructureID(m_node->structure()));
3401         };
3402
3403         m_out.branch(isUnexpectedArray(cell), rarely(unexpectedStructure), usually(continuation));
3404
3405         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
3406         
3407         if (property) {
3408             switch (m_node->arrayMode().type()) {
3409             case Array::Int32:
3410             case Array::Double:
3411             case Array::Contiguous:
3412                 speculate(
3413                     Uncountable, noValue(), 0,
3414                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
3415                 break;
3416             default:
3417                 break;
3418             }
3419         }
3420         
3421         switch (m_node->arrayMode().type()) {
3422         case Array::Int32:
3423             vmCall(Void, m_out.operation(operationEnsureInt32), m_callFrame, cell);
3424             break;
3425         case Array::Double:
3426             vmCall(Void, m_out.operation(operationEnsureDouble), m_callFrame, cell);
3427             break;
3428         case Array::Contiguous:
3429             vmCall(Void, m_out.operation(operationEnsureContiguous), m_callFrame, cell);
3430             break;
3431         case Array::ArrayStorage:
3432         case Array::SlowPutArrayStorage:
3433             vmCall(Void, m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
3434             break;
3435         default:
3436             DFG_CRASH(m_graph, m_node, "Bad array type");
3437             break;
3438         }
3439         
3440         speculate(BadIndexingType, jsValueValue(cell), 0, isUnexpectedArray(cell));
3441         m_out.jump(continuation);
3442         
3443         m_out.appendTo(continuation, lastNext);
3444     }
3445     
3446     void compilePutStructure()
3447     {
3448         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
3449         
3450         RegisteredStructure oldStructure = m_node->transition()->previous;
3451         RegisteredStructure newStructure = m_node->transition()->next;
3452         ASSERT_UNUSED(oldStructure, oldStructure->indexingMode() == newStructure->indexingMode());
3453         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
3454         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
3455
3456         LValue cell = lowCell(m_node->child1()); 
3457         m_out.store32(
3458             weakStructureID(newStructure),
3459             cell, m_heaps.JSCell_structureID);
3460     }
3461     
3462     void compileGetById(AccessType type)
3463     {
3464         ASSERT(type == AccessType::Get || type == AccessType::TryGet || type == AccessType::GetDirect);
3465         switch (m_node->child1().useKind()) {
3466         case CellUse: {
3467             setJSValue(getById(lowCell(m_node->child1()), type));
3468             return;
3469         }
3470             
3471         case UntypedUse: {
3472             // This is pretty weird, since we duplicate the slow path both here and in the
3473             // code generated by the IC. We should investigate making this less bad.
3474             // https://bugs.webkit.org/show_bug.cgi?id=127830
3475             LValue value = lowJSValue(m_node->child1());
3476             
3477             LBasicBlock cellCase = m_out.newBlock();
3478             LBasicBlock notCellCase = m_out.newBlock();
3479             LBasicBlock continuation = m_out.newBlock();
3480             
3481             m_out.branch(
3482                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3483             
3484             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3485             ValueFromBlock cellResult = m_out.anchor(getById(value, type));
3486             m_out.jump(continuation);
3487
3488             J_JITOperation_EJI getByIdFunction = appropriateGenericGetByIdFunction(type);
3489
3490             m_out.appendTo(notCellCase, continuation);
3491             ValueFromBlock notCellResult = m_out.anchor(vmCall(
3492                 Int64, m_out.operation(getByIdFunction),
3493                 m_callFrame, value,
3494                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3495             m_out.jump(continuation);
3496             
3497             m_out.appendTo(continuation, lastNext);
3498             setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3499             return;
3500         }
3501             
3502         default:
3503             DFG_CRASH(m_graph, m_node, "Bad use kind");
3504             return;
3505         }
3506     }
3507
3508     void compileGetByIdWithThis()
3509     {
3510         if (m_node->child1().useKind() == CellUse && m_node->child2().useKind() == CellUse)
3511             setJSValue(getByIdWithThis(lowCell(m_node->child1()), lowCell(m_node->child2())));
3512         else {
3513             LValue base = lowJSValue(m_node->child1());
3514             LValue thisValue = lowJSValue(m_node->child2());
3515             
3516             LBasicBlock baseCellCase = m_out.newBlock();
3517             LBasicBlock notCellCase = m_out.newBlock();
3518             LBasicBlock thisValueCellCase = m_out.newBlock();
3519             LBasicBlock continuation = m_out.newBlock();
3520             
3521             m_out.branch(
3522                 isCell(base, provenType(m_node->child1())), unsure(baseCellCase), unsure(notCellCase));
3523             
3524             LBasicBlock lastNext = m_out.appendTo(baseCellCase, thisValueCellCase);
3525             
3526             m_out.branch(
3527                 isCell(thisValue, provenType(m_node->child2())), unsure(thisValueCellCase), unsure(notCellCase));
3528             
3529             m_out.appendTo(thisValueCellCase, notCellCase);
3530             ValueFromBlock cellResult = m_out.anchor(getByIdWithThis(base, thisValue));
3531             m_out.jump(continuation);
3532
3533             m_out.appendTo(notCellCase, continuation);
3534             ValueFromBlock notCellResult = m_out.anchor(vmCall(
3535                 Int64, m_out.operation(operationGetByIdWithThisGeneric),
3536                 m_callFrame, base, thisValue,
3537                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3538             m_out.jump(continuation);
3539             
3540             m_out.appendTo(continuation, lastNext);
3541             setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3542         }
3543         
3544     }
3545
3546     void compileGetByValWithThis()
3547     {
3548         LValue base = lowJSValue(m_node->child1());
3549         LValue thisValue = lowJSValue(m_node->child2());
3550         LValue subscript = lowJSValue(m_node->child3());
3551
3552         LValue result = vmCall(Int64, m_out.operation(operationGetByValWithThis), m_callFrame, base, thisValue, subscript);
3553         setJSValue(result);
3554     }
3555
3556     void compilePutByIdWithThis()
3557     {
3558         LValue base = lowJSValue(m_node->child1());
3559         LValue thisValue = lowJSValue(m_node->child2());
3560         LValue value = lowJSValue(m_node->child3());
3561
3562         vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis),
3563             m_callFrame, base, thisValue, value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()]));
3564     }
3565
3566     void compilePutByValWithThis()
3567     {
3568         LValue base = lowJSValue(m_graph.varArgChild(m_node, 0));
3569         LValue thisValue = lowJSValue(m_graph.varArgChild(m_node, 1));
3570         LValue property = lowJSValue(m_graph.varArgChild(m_node, 2));
3571         LValue value = lowJSValue(m_graph.varArgChild(m_node, 3));
3572
3573         vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis),
3574             m_callFrame, base, thisValue, property, value);
3575     }
3576     
3577     void compileAtomicsReadModifyWrite()
3578     {
3579         TypedArrayType type = m_node->arrayMode().typedArrayType();
3580         unsigned numExtraArgs = numExtraAtomicsArgs(m_node->op());
3581         Edge baseEdge = m_graph.child(m_node, 0);
3582         Edge indexEdge = m_graph.child(m_node, 1);
3583         Edge argEdges[maxNumExtraAtomicsArgs];
3584         for (unsigned i = numExtraArgs; i--;)
3585             argEdges[i] = m_graph.child(m_node, 2 + i);
3586         Edge storageEdge = m_graph.child(m_node, 2 + numExtraArgs);
3587         
3588         auto operation = [&] () -> LValue {
3589             switch (m_node->op()) {
3590             case AtomicsAdd:
3591                 return m_out.operation(operationAtomicsAdd);
3592             case AtomicsAnd:
3593                 return m_out.operation(operationAtomicsAnd);
3594             case AtomicsCompareExchange:
3595                 return m_out.operation(operationAtomicsCompareExchange);
3596             case AtomicsExchange:
3597                 return m_out.operation(operationAtomicsExchange);
3598             case AtomicsLoad:
3599                 return m_out.operation(operationAtomicsLoad);
3600             case AtomicsOr:
3601                 return m_out.operation(operationAtomicsOr);
3602             case AtomicsStore:
3603                 return m_out.operation(operationAtomicsStore);
3604             case AtomicsSub:
3605                 return m_out.operation(operationAtomicsSub);
3606             case AtomicsXor:
3607                 return m_out.operation(operationAtomicsXor);
3608             default:
3609                 RELEASE_ASSERT_NOT_REACHED();
3610                 break;
3611             }
3612         };
3613         
3614         if (!storageEdge) {
3615             Vector<LValue> args;
3616             args.append(m_callFrame);
3617             args.append(lowJSValue(baseEdge));
3618             args.append(lowJSValue(indexEdge));
3619             for (unsigned i = 0; i < numExtraArgs; ++i)
3620                 args.append(lowJSValue(argEdges[i]));
3621             LValue result = vmCall(Int64, operation(), args);
3622             setJSValue(result);
3623             return;
3624         }
3625
3626         LValue index = lowInt32(indexEdge);
3627         LValue args[2];
3628         for (unsigned i = numExtraArgs; i--;)
3629             args[i] = getIntTypedArrayStoreOperand(argEdges[i]);
3630         LValue storage = lowStorage(storageEdge);
3631
3632         TypedPointer pointer = pointerIntoTypedArray(storage, index, type);