6c6574863bd14d65b31401ae7fa403b35ac90665
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToB3.cpp
1 /*
2  * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToB3.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "AirCode.h"
32 #include "AirGenerationContext.h"
33 #include "AllowMacroScratchRegisterUsage.h"
34 #include "AllowMacroScratchRegisterUsageIf.h"
35 #include "AtomicsObject.h"
36 #include "B3CheckValue.h"
37 #include "B3FenceValue.h"
38 #include "B3PatchpointValue.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3ValueInlines.h"
42 #include "CallFrameShuffler.h"
43 #include "CodeBlockWithJITType.h"
44 #include "DFGAbstractInterpreterInlines.h"
45 #include "DFGCapabilities.h"
46 #include "DFGDoesGC.h"
47 #include "DFGDominators.h"
48 #include "DFGInPlaceAbstractState.h"
49 #include "DFGLivenessAnalysisPhase.h"
50 #include "DFGMayExit.h"
51 #include "DFGOSRAvailabilityAnalysisPhase.h"
52 #include "DFGOSRExitFuzz.h"
53 #include "DirectArguments.h"
54 #include "FTLAbstractHeapRepository.h"
55 #include "FTLAvailableRecovery.h"
56 #include "FTLExceptionTarget.h"
57 #include "FTLForOSREntryJITCode.h"
58 #include "FTLFormattedValue.h"
59 #include "FTLLazySlowPathCall.h"
60 #include "FTLLoweredNodeValue.h"
61 #include "FTLOperations.h"
62 #include "FTLOutput.h"
63 #include "FTLPatchpointExceptionHandle.h"
64 #include "FTLSnippetParams.h"
65 #include "FTLThunks.h"
66 #include "FTLWeightedTarget.h"
67 #include "JITAddGenerator.h"
68 #include "JITBitAndGenerator.h"
69 #include "JITBitOrGenerator.h"
70 #include "JITBitXorGenerator.h"
71 #include "JITDivGenerator.h"
72 #include "JITInlineCacheGenerator.h"
73 #include "JITLeftShiftGenerator.h"
74 #include "JITMathIC.h"
75 #include "JITMulGenerator.h"
76 #include "JITRightShiftGenerator.h"
77 #include "JITSubGenerator.h"
78 #include "JSAsyncFunction.h"
79 #include "JSAsyncGeneratorFunction.h"
80 #include "JSCInlines.h"
81 #include "JSGeneratorFunction.h"
82 #include "JSImmutableButterfly.h"
83 #include "JSLexicalEnvironment.h"
84 #include "JSMap.h"
85 #include "OperandsInlines.h"
86 #include "ProbeContext.h"
87 #include "RegExpObject.h"
88 #include "ScopedArguments.h"
89 #include "ScopedArgumentsTable.h"
90 #include "ScratchRegisterAllocator.h"
91 #include "SetupVarargsFrame.h"
92 #include "ShadowChicken.h"
93 #include "StructureStubInfo.h"
94 #include "SuperSampler.h"
95 #include "ThunkGenerators.h"
96 #include "VirtualRegister.h"
97 #include "Watchdog.h"
98 #include <atomic>
99 #include <wtf/Box.h>
100 #include <wtf/Gigacage.h>
101 #include <wtf/RecursableLambda.h>
102 #include <wtf/StdUnorderedSet.h>
103
104 #undef RELEASE_ASSERT
105 #define RELEASE_ASSERT(assertion) do { \
106     if (!(assertion)) { \
107         WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #assertion); \
108         CRASH(); \
109     } \
110 } while (0)
111
112 namespace JSC { namespace FTL {
113
114 using namespace B3;
115 using namespace DFG;
116
117 namespace {
118
119 std::atomic<int> compileCounter;
120
121 #if !ASSERT_DISABLED
122 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
123     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
124 {
125     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
126     if (nodeIndex != UINT_MAX)
127         dataLog(", node @", nodeIndex);
128     dataLog(".\n");
129     CRASH();
130 }
131 #endif
132
133 // Using this instead of typeCheck() helps to reduce the load on B3, by creating
134 // significantly less dead code.
135 #define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \
136         FormattedValue _ftc_lowValue = (lowValue);                      \
137         Edge _ftc_highValue = (highValue);                              \
138         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
139         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
140             break;                                                      \
141         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \
142     } while (false)
143
144 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \
145     FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition)
146
147 class LowerDFGToB3 {
148     WTF_MAKE_NONCOPYABLE(LowerDFGToB3);
149 public:
150     LowerDFGToB3(State& state)
151         : m_graph(state.graph)
152         , m_ftlState(state)
153         , m_out(state)
154         , m_proc(*state.proc)
155         , m_availabilityCalculator(m_graph)
156         , m_state(state.graph)
157         , m_interpreter(state.graph, m_state)
158         , m_indexMaskingMode(Options::enableSpectreMitigations() ?  IndexMaskingEnabled : IndexMaskingDisabled)
159     {
160         if (Options::validateAbstractInterpreterState()) {
161             performLivenessAnalysis(m_graph);
162
163             // We only use node liveness here, not combined liveness, as we only track
164             // AI state for live nodes.
165             for (DFG::BasicBlock* block : m_graph.blocksInNaturalOrder()) {
166                 NodeSet live;
167
168                 for (NodeFlowProjection node : block->ssa->liveAtTail) {
169                     if (node.kind() == NodeFlowProjection::Primary)
170                         live.addVoid(node.node());
171                 }
172
173                 for (unsigned i = block->size(); i--; ) {
174                     Node* node = block->at(i);
175                     live.remove(node);
176                     m_graph.doToChildren(node, [&] (Edge child) {
177                         live.addVoid(child.node());
178                     });
179                     m_liveInToNode.add(node, live);
180                 }
181             }
182         }
183     }
184     
185     void lower()
186     {
187         State* state = &m_ftlState;
188         
189         CString name;
190         if (verboseCompilationEnabled()) {
191             name = toCString(
192                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
193                 "_", codeBlock()->hash());
194         } else
195             name = "jsBody";
196
197         {
198             m_proc.setNumEntrypoints(m_graph.m_numberOfEntrypoints);
199             CodeBlock* codeBlock = m_graph.m_codeBlock;
200
201             Ref<B3::Air::PrologueGenerator> catchPrologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>(
202                 [codeBlock] (CCallHelpers& jit, B3::Air::Code& code) {
203                     AllowMacroScratchRegisterUsage allowScratch(jit);
204                     jit.addPtr(CCallHelpers::TrustedImm32(-code.frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
205                     if (Options::zeroStackFrame())
206                         jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, code.frameSize());
207
208                     jit.emitSave(code.calleeSaveRegisterAtOffsetList());
209                     jit.emitPutToCallFrameHeader(codeBlock, CallFrameSlot::codeBlock);
210                 });
211
212             for (unsigned catchEntrypointIndex : m_graph.m_entrypointIndexToCatchBytecodeOffset.keys()) {
213                 RELEASE_ASSERT(catchEntrypointIndex != 0);
214                 m_proc.code().setPrologueForEntrypoint(catchEntrypointIndex, catchPrologueGenerator.copyRef());
215             }
216
217             if (m_graph.m_maxLocalsForCatchOSREntry) {
218                 uint32_t numberOfLiveLocals = std::max(*m_graph.m_maxLocalsForCatchOSREntry, 1u); // Make sure we always allocate a non-null catchOSREntryBuffer.
219                 m_ftlState.jitCode->common.catchOSREntryBuffer = m_graph.m_vm.scratchBufferForSize(sizeof(JSValue) * numberOfLiveLocals);
220             }
221         }
222         
223         m_graph.ensureSSADominators();
224
225         if (verboseCompilationEnabled())
226             dataLog("Function ready, beginning lowering.\n");
227
228         m_out.initialize(m_heaps);
229
230         // We use prologue frequency for all of the initialization code.
231         m_out.setFrequency(1);
232         
233         bool hasMultipleEntrypoints = m_graph.m_numberOfEntrypoints > 1;
234     
235         LBasicBlock prologue = m_out.newBlock();
236         LBasicBlock callEntrypointArgumentSpeculations = hasMultipleEntrypoints ? m_out.newBlock() : nullptr;
237         m_handleExceptions = m_out.newBlock();
238
239         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
240             m_highBlock = m_graph.block(blockIndex);
241             if (!m_highBlock)
242                 continue;
243             m_out.setFrequency(m_highBlock->executionCount);
244             m_blocks.add(m_highBlock, m_out.newBlock());
245         }
246
247         // Back to prologue frequency for any bocks that get sneakily created in the initialization code.
248         m_out.setFrequency(1);
249         
250         m_out.appendTo(prologue, hasMultipleEntrypoints ? callEntrypointArgumentSpeculations : m_handleExceptions);
251         m_out.initializeConstants(m_proc, prologue);
252         createPhiVariables();
253
254         size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;
255         B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);
256         m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));
257         state->capturedValue = capturedBase->slot();
258
259         auto preOrder = m_graph.blocksInPreOrder();
260
261         m_callFrame = m_out.framePointer();
262         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
263         m_tagMask = m_out.constInt64(TagMask);
264
265         // Make sure that B3 knows that we really care about the mask registers. This forces the
266         // constants to be materialized in registers.
267         m_proc.addFastConstant(m_tagTypeNumber->key());
268         m_proc.addFastConstant(m_tagMask->key());
269         
270         // We don't want the CodeBlock to have a weak pointer to itself because
271         // that would cause it to always get collected.
272         m_out.storePtr(m_out.constIntPtr(bitwise_cast<intptr_t>(codeBlock())), addressFor(CallFrameSlot::codeBlock));
273
274         VM* vm = &this->vm();
275
276         // Stack Overflow Check.
277         unsigned exitFrameSize = m_graph.requiredRegisterCountForExit() * sizeof(Register);
278         MacroAssembler::AbsoluteAddress addressOfStackLimit(vm->addressOfSoftStackLimit());
279         PatchpointValue* stackOverflowHandler = m_out.patchpoint(Void);
280         CallSiteIndex callSiteIndex = callSiteIndexForCodeOrigin(m_ftlState, CodeOrigin(0));
281         stackOverflowHandler->appendSomeRegister(m_callFrame);
282         stackOverflowHandler->clobber(RegisterSet::macroScratchRegisters());
283         stackOverflowHandler->numGPScratchRegisters = 1;
284         stackOverflowHandler->setGenerator(
285             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
286                 AllowMacroScratchRegisterUsage allowScratch(jit);
287                 GPRReg fp = params[0].gpr();
288                 GPRReg scratch = params.gpScratch(0);
289
290                 unsigned ftlFrameSize = params.proc().frameSize();
291                 unsigned maxFrameSize = std::max(exitFrameSize, ftlFrameSize);
292
293                 jit.addPtr(MacroAssembler::TrustedImm32(-maxFrameSize), fp, scratch);
294                 MacroAssembler::JumpList stackOverflow;
295                 if (UNLIKELY(maxFrameSize > Options::reservedZoneSize()))
296                     stackOverflow.append(jit.branchPtr(MacroAssembler::Above, scratch, fp));
297                 stackOverflow.append(jit.branchPtr(MacroAssembler::Above, addressOfStackLimit, scratch));
298
299                 params.addLatePath([=] (CCallHelpers& jit) {
300                     AllowMacroScratchRegisterUsage allowScratch(jit);
301
302                     stackOverflow.link(&jit);
303                     
304                     // FIXME: We would not have to do this if the stack check was part of the Air
305                     // prologue. Then, we would know that there is no way for the callee-saves to
306                     // get clobbered.
307                     // https://bugs.webkit.org/show_bug.cgi?id=172456
308                     jit.emitRestore(params.proc().calleeSaveRegisterAtOffsetList());
309                     
310                     jit.store32(
311                         MacroAssembler::TrustedImm32(callSiteIndex.bits()),
312                         CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
313                     jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm->topEntryFrame);
314
315                     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
316                     jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()), GPRInfo::argumentGPR1);
317                     CCallHelpers::Call throwCall = jit.call(OperationPtrTag);
318
319                     jit.move(CCallHelpers::TrustedImmPtr(vm), GPRInfo::argumentGPR0);
320                     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
321                     CCallHelpers::Call lookupExceptionHandlerCall = jit.call(OperationPtrTag);
322                     jit.jumpToExceptionHandler(*vm);
323
324                     jit.addLinkTask(
325                         [=] (LinkBuffer& linkBuffer) {
326                             linkBuffer.link(throwCall, FunctionPtr<OperationPtrTag>(operationThrowStackOverflowError));
327                             linkBuffer.link(lookupExceptionHandlerCall, FunctionPtr<OperationPtrTag>(lookupExceptionHandlerFromCallerFrame));
328                     });
329                 });
330             });
331
332         LBasicBlock firstDFGBasicBlock = lowBlock(m_graph.block(0));
333
334         {
335             if (hasMultipleEntrypoints) {
336                 Vector<LBasicBlock> successors(m_graph.m_numberOfEntrypoints);
337                 successors[0] = callEntrypointArgumentSpeculations;
338                 for (unsigned i = 1; i < m_graph.m_numberOfEntrypoints; ++i) {
339                     // Currently, the only other entrypoint is an op_catch entrypoint.
340                     // We do OSR entry at op_catch, and we prove argument formats before
341                     // jumping to FTL code, so we don't need to check argument types here
342                     // for these entrypoints.
343                     successors[i] = firstDFGBasicBlock;
344                 }
345                 
346                 m_out.entrySwitch(successors);
347                 m_out.appendTo(callEntrypointArgumentSpeculations, m_handleExceptions);
348             }
349
350             m_node = nullptr;
351             m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
352
353             // Check Arguments.
354             availabilityMap().clear();
355             availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
356             for (unsigned i = codeBlock()->numParameters(); i--;) {
357                 availabilityMap().m_locals.argument(i) =
358                     Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
359             }
360
361             for (unsigned i = codeBlock()->numParameters(); i--;) {
362                 MethodOfGettingAValueProfile profile(&m_graph.m_profiledBlock->valueProfileForArgument(i));
363                 VirtualRegister operand = virtualRegisterForArgument(i);
364                 LValue jsValue = m_out.load64(addressFor(operand));
365                 
366                 switch (m_graph.m_argumentFormats[0][i]) {
367                 case FlushedInt32:
368                     speculate(BadType, jsValueValue(jsValue), profile, isNotInt32(jsValue));
369                     break;
370                 case FlushedBoolean:
371                     speculate(BadType, jsValueValue(jsValue), profile, isNotBoolean(jsValue));
372                     break;
373                 case FlushedCell:
374                     speculate(BadType, jsValueValue(jsValue), profile, isNotCell(jsValue));
375                     break;
376                 case FlushedJSValue:
377                     break;
378                 default:
379                     DFG_CRASH(m_graph, nullptr, "Bad flush format for argument");
380                     break;
381                 }
382             }
383             m_out.jump(firstDFGBasicBlock);
384         }
385
386
387         m_out.appendTo(m_handleExceptions, firstDFGBasicBlock);
388         Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;
389         m_out.patchpoint(Void)->setGenerator(
390             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
391                 CCallHelpers::Jump jump = jit.jump();
392                 jit.addLinkTask(
393                     [=] (LinkBuffer& linkBuffer) {
394                         linkBuffer.link(jump, linkBuffer.locationOf<ExceptionHandlerPtrTag>(*exceptionHandler));
395                     });
396             });
397         m_out.unreachable();
398
399         for (DFG::BasicBlock* block : preOrder)
400             compileBlock(block);
401
402         // Make sure everything is decorated. This does a bunch of deferred decorating. This has
403         // to happen last because our abstract heaps are generated lazily. They have to be
404         // generated lazily because we have an infinite number of numbered, indexed, and
405         // absolute heaps. We only become aware of the ones we actually mention while lowering.
406         m_heaps.computeRangesAndDecorateInstructions();
407
408         // We create all Phi's up front, but we may then decide not to compile the basic block
409         // that would have contained one of them. So this creates orphans, which triggers B3
410         // validation failures. Calling this fixes the issue.
411         //
412         // Note that you should avoid the temptation to make this call conditional upon
413         // validation being enabled. B3 makes no guarantees of any kind of correctness when
414         // dealing with IR that would have failed validation. For example, it would be valid to
415         // write a B3 phase that so aggressively assumes the lack of orphans that it would crash
416         // if any orphans were around. We might even have such phases already.
417         m_proc.deleteOrphans();
418
419         // We put the blocks into the B3 procedure in a super weird order. Now we reorder them.
420         m_out.applyBlockOrder();
421     }
422
423 private:
424     
425     void createPhiVariables()
426     {
427         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
428             DFG::BasicBlock* block = m_graph.block(blockIndex);
429             if (!block)
430                 continue;
431             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
432                 Node* node = block->at(nodeIndex);
433                 if (node->op() != DFG::Phi)
434                     continue;
435                 LType type;
436                 switch (node->flags() & NodeResultMask) {
437                 case NodeResultDouble:
438                     type = Double;
439                     break;
440                 case NodeResultInt32:
441                     type = Int32;
442                     break;
443                 case NodeResultInt52:
444                     type = Int64;
445                     break;
446                 case NodeResultBoolean:
447                     type = Int32;
448                     break;
449                 case NodeResultJS:
450                     type = Int64;
451                     break;
452                 default:
453                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
454                     break;
455                 }
456                 m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));
457             }
458         }
459     }
460     
461     void compileBlock(DFG::BasicBlock* block)
462     {
463         if (!block)
464             return;
465         
466         if (verboseCompilationEnabled())
467             dataLog("Compiling block ", *block, "\n");
468         
469         m_highBlock = block;
470         
471         // Make sure that any blocks created while lowering code in the high block have the frequency of
472         // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs
473         // something roughly approximate for things like register allocation.
474         m_out.setFrequency(m_highBlock->executionCount);
475         
476         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
477         
478         m_nextHighBlock = 0;
479         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
480             m_nextHighBlock = m_graph.block(nextBlockIndex);
481             if (m_nextHighBlock)
482                 break;
483         }
484         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
485         
486         // All of this effort to find the next block gives us the ability to keep the
487         // generated IR in roughly program order. This ought not affect the performance
488         // of the generated code (since we expect B3 to reorder things) but it will
489         // make IR dumps easier to read.
490         m_out.appendTo(lowBlock, m_nextLowBlock);
491         
492         if (Options::ftlCrashes())
493             m_out.trap();
494         
495         if (!m_highBlock->cfaHasVisited) {
496             if (verboseCompilationEnabled())
497                 dataLog("Bailing because CFA didn't reach.\n");
498             crash(m_highBlock, nullptr);
499             return;
500         }
501
502         m_aiCheckedNodes.clear();
503         
504         m_availabilityCalculator.beginBlock(m_highBlock);
505         
506         m_state.reset();
507         m_state.beginBasicBlock(m_highBlock);
508         
509         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
510             if (!compileNode(m_nodeIndex))
511                 break;
512         }
513     }
514
515     void safelyInvalidateAfterTermination()
516     {
517         if (verboseCompilationEnabled())
518             dataLog("Bailing.\n");
519         crash();
520
521         // Invalidate dominated blocks. Under normal circumstances we would expect
522         // them to be invalidated already. But you can have the CFA become more
523         // precise over time because the structures of objects change on the main
524         // thread. Failing to do this would result in weird crashes due to a value
525         // being used but not defined. Race conditions FTW!
526         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
527             DFG::BasicBlock* target = m_graph.block(blockIndex);
528             if (!target)
529                 continue;
530             if (m_graph.m_ssaDominators->dominates(m_highBlock, target)) {
531                 if (verboseCompilationEnabled())
532                     dataLog("Block ", *target, " will bail also.\n");
533                 target->cfaHasVisited = false;
534             }
535         }
536     }
537
538     void validateAIState(Node* node)
539     {
540         if (!m_graphDump) {
541             StringPrintStream out;
542             m_graph.dump(out);
543             m_graphDump = out.toString();
544         }
545
546         switch (node->op()) {
547         case MovHint:
548         case ZombieHint:
549         case JSConstant:
550         case LazyJSConstant:
551         case DoubleConstant:
552         case Int52Constant:
553         case GetStack:
554         case PutStack:
555         case KillStack:
556         case ExitOK:
557             return;
558         default:
559             break;
560         }
561
562         // Before we execute node.
563         NodeSet& live = m_liveInToNode.find(node)->value;
564         unsigned highParentIndex = node->index();
565         {
566             uint64_t hash = WTF::intHash(highParentIndex);
567             if (hash >= static_cast<uint64_t>((static_cast<double>(std::numeric_limits<unsigned>::max()) + 1) * Options::validateAbstractInterpreterStateProbability()))
568                 return;
569         }
570
571         for (Node* node : live) {
572             if (node->isPhantomAllocation())
573                 continue;
574
575             if (node->op() == CheckInBounds)
576                 continue;
577
578             AbstractValue value = m_interpreter.forNode(node);
579             {
580                 auto iter = m_aiCheckedNodes.find(node);
581                 if (iter != m_aiCheckedNodes.end()) {
582                     AbstractValue checkedValue = iter->value;
583                     if (checkedValue == value) {
584                         if (!(value.m_type & SpecCell))
585                             continue;
586                     }
587                 }
588                 m_aiCheckedNodes.set(node, value);
589             }
590
591             FlushFormat flushFormat;
592             LValue input;
593             if (node->hasJSResult()) {
594                 input = lowJSValue(Edge(node, UntypedUse));
595                 flushFormat = FlushedJSValue;
596             } else if (node->hasDoubleResult()) {
597                 input = lowDouble(Edge(node, DoubleRepUse));
598                 flushFormat = FlushedDouble;
599             } else if (node->hasInt52Result()) {
600                 input = strictInt52ToJSValue(lowStrictInt52(Edge(node, Int52RepUse)));
601                 flushFormat = FlushedInt52;
602             } else
603                 continue;
604
605             unsigned highChildIndex = node->index();
606
607             String graphDump = m_graphDump;
608
609             PatchpointValue* patchpoint = m_out.patchpoint(Void);
610             patchpoint->effects = Effects::none();
611             patchpoint->effects.writesLocalState = true;
612             patchpoint->appendSomeRegister(input);
613             patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
614                 GPRReg reg = InvalidGPRReg;
615                 FPRReg fpReg = InvalidFPRReg;
616                 if (flushFormat == FlushedDouble)
617                     fpReg = params[0].fpr();
618                 else
619                     reg = params[0].gpr();
620                 jit.probe([=] (Probe::Context& context) {
621                     JSValue input;
622                     double doubleInput;
623
624                     auto dumpAndCrash = [&] {
625                         dataLogLn("Validation failed at node: @", highParentIndex);
626                         dataLogLn("Failed validating live value: @", highChildIndex);
627                         dataLogLn();
628                         dataLogLn("Expected AI value = ", value);
629                         if (flushFormat != FlushedDouble)
630                             dataLogLn("Unexpected value = ", input);
631                         else
632                             dataLogLn("Unexpected double value = ", doubleInput);
633                         dataLogLn();
634                         dataLogLn(graphDump);
635                         CRASH();
636                     };
637
638                     if (flushFormat == FlushedDouble) {
639                         doubleInput = context.fpr(fpReg);
640                         SpeculatedType type;
641                         if (!std::isnan(doubleInput))
642                             type = speculationFromValue(jsDoubleNumber(doubleInput));
643                         else if (isImpureNaN(doubleInput))
644                             type = SpecDoubleImpureNaN;
645                         else
646                             type = SpecDoublePureNaN;
647
648                         if (!value.couldBeType(type))
649                             dumpAndCrash();
650                     } else {
651                         input = JSValue::decode(context.gpr(reg)); 
652                         if (flushFormat == FlushedInt52) {
653                             RELEASE_ASSERT(input.isAnyInt());
654                             input = jsDoubleNumber(input.asAnyInt());
655                         }
656                         if (!value.validateOSREntryValue(input, flushFormat))
657                             dumpAndCrash();
658                     }
659
660                 });
661             });
662         }
663     }
664
665     bool compileNode(unsigned nodeIndex)
666     {
667         if (!m_state.isValid()) {
668             safelyInvalidateAfterTermination();
669             return false;
670         }
671         
672         m_node = m_highBlock->at(nodeIndex);
673         m_origin = m_node->origin;
674         m_out.setOrigin(m_node);
675         
676         if (verboseCompilationEnabled())
677             dataLog("Lowering ", m_node, "\n");
678         
679         m_interpreter.startExecuting();
680         m_interpreter.executeKnownEdgeTypes(m_node);
681
682         if (Options::validateAbstractInterpreterState())
683             validateAIState(m_node);
684
685         if (validateDFGDoesGC) {
686             bool expectDoesGC = doesGC(m_graph, m_node);
687             m_out.store(m_out.constBool(expectDoesGC), m_out.absolute(vm().heap.addressOfExpectDoesGC()));
688         }
689
690         switch (m_node->op()) {
691         case DFG::Upsilon:
692             compileUpsilon();
693             break;
694         case DFG::Phi:
695             compilePhi();
696             break;
697         case JSConstant:
698             break;
699         case DoubleConstant:
700             compileDoubleConstant();
701             break;
702         case Int52Constant:
703             compileInt52Constant();
704             break;
705         case LazyJSConstant:
706             compileLazyJSConstant();
707             break;
708         case DoubleRep:
709             compileDoubleRep();
710             break;
711         case DoubleAsInt32:
712             compileDoubleAsInt32();
713             break;
714         case DFG::ValueRep:
715             compileValueRep();
716             break;
717         case Int52Rep:
718             compileInt52Rep();
719             break;
720         case ValueToInt32:
721             compileValueToInt32();
722             break;
723         case BooleanToNumber:
724             compileBooleanToNumber();
725             break;
726         case ExtractOSREntryLocal:
727             compileExtractOSREntryLocal();
728             break;
729         case ExtractCatchLocal:
730             compileExtractCatchLocal();
731             break;
732         case ClearCatchLocals:
733             compileClearCatchLocals();
734             break;
735         case GetStack:
736             compileGetStack();
737             break;
738         case PutStack:
739             compilePutStack();
740             break;
741         case DFG::Check:
742         case CheckVarargs:
743             compileNoOp();
744             break;
745         case ToObject:
746         case CallObjectConstructor:
747             compileToObjectOrCallObjectConstructor();
748             break;
749         case ToThis:
750             compileToThis();
751             break;
752         case ValueNegate:
753             compileValueNegate();
754             break;
755         case ValueAdd:
756             compileValueAdd();
757             break;
758         case ValueSub:
759             compileValueSub();
760             break;
761         case ValueMul:
762             compileValueMul();
763             break;
764         case StrCat:
765             compileStrCat();
766             break;
767         case ArithAdd:
768         case ArithSub:
769             compileArithAddOrSub();
770             break;
771         case ArithClz32:
772             compileArithClz32();
773             break;
774         case ArithMul:
775             compileArithMul();
776             break;
777         case ValueDiv:
778             compileValueDiv();
779             break;
780         case ArithDiv:
781             compileArithDiv();
782             break;
783         case ValueMod:
784             compileValueMod();
785             break;
786         case ArithMod:
787             compileArithMod();
788             break;
789         case ArithMin:
790         case ArithMax:
791             compileArithMinOrMax();
792             break;
793         case ArithAbs:
794             compileArithAbs();
795             break;
796         case ArithPow:
797             compileArithPow();
798             break;
799         case ArithRandom:
800             compileArithRandom();
801             break;
802         case ArithRound:
803             compileArithRound();
804             break;
805         case ArithFloor:
806             compileArithFloor();
807             break;
808         case ArithCeil:
809             compileArithCeil();
810             break;
811         case ArithTrunc:
812             compileArithTrunc();
813             break;
814         case ArithSqrt:
815             compileArithSqrt();
816             break;
817         case ArithFRound:
818             compileArithFRound();
819             break;
820         case ArithNegate:
821             compileArithNegate();
822             break;
823         case ArithUnary:
824             compileArithUnary();
825             break;
826         case ValueBitNot:
827             compileValueBitNot();
828             break;
829         case ArithBitNot:
830             compileArithBitNot();
831             break;
832         case ValueBitAnd:
833             compileValueBitAnd();
834             break;
835         case ArithBitAnd:
836             compileArithBitAnd();
837             break;
838         case ValueBitOr:
839             compileValueBitOr();
840             break;
841         case ArithBitOr:
842             compileArithBitOr();
843             break;
844         case ArithBitXor:
845             compileArithBitXor();
846             break;
847         case ValueBitXor:
848             compileValueBitXor();
849             break;
850         case BitRShift:
851             compileBitRShift();
852             break;
853         case BitLShift:
854             compileBitLShift();
855             break;
856         case BitURShift:
857             compileBitURShift();
858             break;
859         case UInt32ToNumber:
860             compileUInt32ToNumber();
861             break;
862         case CheckStructure:
863             compileCheckStructure();
864             break;
865         case CheckStructureOrEmpty:
866             compileCheckStructureOrEmpty();
867             break;
868         case CheckCell:
869             compileCheckCell();
870             break;
871         case CheckNotEmpty:
872             compileCheckNotEmpty();
873             break;
874         case AssertNotEmpty:
875             compileAssertNotEmpty();
876             break;
877         case CheckBadCell:
878             compileCheckBadCell();
879             break;
880         case CheckStringIdent:
881             compileCheckStringIdent();
882             break;
883         case GetExecutable:
884             compileGetExecutable();
885             break;
886         case Arrayify:
887         case ArrayifyToStructure:
888             compileArrayify();
889             break;
890         case PutStructure:
891             compilePutStructure();
892             break;
893         case TryGetById:
894             compileGetById(AccessType::TryGet);
895             break;
896         case GetById:
897         case GetByIdFlush:
898             compileGetById(AccessType::Get);
899             break;
900         case GetByIdWithThis:
901             compileGetByIdWithThis();
902             break;
903         case GetByIdDirect:
904         case GetByIdDirectFlush:
905             compileGetById(AccessType::GetDirect);
906             break;
907         case InById:
908             compileInById();
909             break;
910         case InByVal:
911             compileInByVal();
912             break;
913         case HasOwnProperty:
914             compileHasOwnProperty();
915             break;
916         case PutById:
917         case PutByIdDirect:
918         case PutByIdFlush:
919             compilePutById();
920             break;
921         case PutByIdWithThis:
922             compilePutByIdWithThis();
923             break;
924         case PutGetterById:
925         case PutSetterById:
926             compilePutAccessorById();
927             break;
928         case PutGetterSetterById:
929             compilePutGetterSetterById();
930             break;
931         case PutGetterByVal:
932         case PutSetterByVal:
933             compilePutAccessorByVal();
934             break;
935         case DeleteById:
936             compileDeleteById();
937             break;
938         case DeleteByVal:
939             compileDeleteByVal();
940             break;
941         case GetButterfly:
942             compileGetButterfly();
943             break;
944         case ConstantStoragePointer:
945             compileConstantStoragePointer();
946             break;
947         case GetIndexedPropertyStorage:
948             compileGetIndexedPropertyStorage();
949             break;
950         case CheckArray:
951             compileCheckArray();
952             break;
953         case GetArrayLength:
954             compileGetArrayLength();
955             break;
956         case GetVectorLength:
957             compileGetVectorLength();
958             break;
959         case CheckInBounds:
960             compileCheckInBounds();
961             break;
962         case GetByVal:
963             compileGetByVal();
964             break;
965         case GetMyArgumentByVal:
966         case GetMyArgumentByValOutOfBounds:
967             compileGetMyArgumentByVal();
968             break;
969         case GetByValWithThis:
970             compileGetByValWithThis();
971             break;
972         case PutByVal:
973         case PutByValAlias:
974         case PutByValDirect:
975             compilePutByVal();
976             break;
977         case PutByValWithThis:
978             compilePutByValWithThis();
979             break;
980         case AtomicsAdd:
981         case AtomicsAnd:
982         case AtomicsCompareExchange:
983         case AtomicsExchange:
984         case AtomicsLoad:
985         case AtomicsOr:
986         case AtomicsStore:
987         case AtomicsSub:
988         case AtomicsXor:
989             compileAtomicsReadModifyWrite();
990             break;
991         case AtomicsIsLockFree:
992             compileAtomicsIsLockFree();
993             break;
994         case DefineDataProperty:
995             compileDefineDataProperty();
996             break;
997         case DefineAccessorProperty:
998             compileDefineAccessorProperty();
999             break;
1000         case ArrayPush:
1001             compileArrayPush();
1002             break;
1003         case ArrayPop:
1004             compileArrayPop();
1005             break;
1006         case ArraySlice:
1007             compileArraySlice();
1008             break;
1009         case ArrayIndexOf:
1010             compileArrayIndexOf();
1011             break;
1012         case CreateActivation:
1013             compileCreateActivation();
1014             break;
1015         case PushWithScope:
1016             compilePushWithScope();
1017             break;
1018         case NewFunction:
1019         case NewGeneratorFunction:
1020         case NewAsyncGeneratorFunction:
1021         case NewAsyncFunction:
1022             compileNewFunction();
1023             break;
1024         case CreateDirectArguments:
1025             compileCreateDirectArguments();
1026             break;
1027         case CreateScopedArguments:
1028             compileCreateScopedArguments();
1029             break;
1030         case CreateClonedArguments:
1031             compileCreateClonedArguments();
1032             break;
1033         case ObjectCreate:
1034             compileObjectCreate();
1035             break;
1036         case ObjectKeys:
1037             compileObjectKeys();
1038             break;
1039         case NewObject:
1040             compileNewObject();
1041             break;
1042         case NewStringObject:
1043             compileNewStringObject();
1044             break;
1045         case NewSymbol:
1046             compileNewSymbol();
1047             break;
1048         case NewArray:
1049             compileNewArray();
1050             break;
1051         case NewArrayWithSpread:
1052             compileNewArrayWithSpread();
1053             break;
1054         case CreateThis:
1055             compileCreateThis();
1056             break;
1057         case Spread:
1058             compileSpread();
1059             break;
1060         case NewArrayBuffer:
1061             compileNewArrayBuffer();
1062             break;
1063         case NewArrayWithSize:
1064             compileNewArrayWithSize();
1065             break;
1066         case NewTypedArray:
1067             compileNewTypedArray();
1068             break;
1069         case GetTypedArrayByteOffset:
1070             compileGetTypedArrayByteOffset();
1071             break;
1072         case GetPrototypeOf:
1073             compileGetPrototypeOf();
1074             break;
1075         case AllocatePropertyStorage:
1076             compileAllocatePropertyStorage();
1077             break;
1078         case ReallocatePropertyStorage:
1079             compileReallocatePropertyStorage();
1080             break;
1081         case NukeStructureAndSetButterfly:
1082             compileNukeStructureAndSetButterfly();
1083             break;
1084         case ToNumber:
1085             compileToNumber();
1086             break;
1087         case ToString:
1088         case CallStringConstructor:
1089         case StringValueOf:
1090             compileToStringOrCallStringConstructorOrStringValueOf();
1091             break;
1092         case ToPrimitive:
1093             compileToPrimitive();
1094             break;
1095         case MakeRope:
1096             compileMakeRope();
1097             break;
1098         case StringCharAt:
1099             compileStringCharAt();
1100             break;
1101         case StringCharCodeAt:
1102             compileStringCharCodeAt();
1103             break;
1104         case StringFromCharCode:
1105             compileStringFromCharCode();
1106             break;
1107         case GetByOffset:
1108         case GetGetterSetterByOffset:
1109             compileGetByOffset();
1110             break;
1111         case GetGetter:
1112             compileGetGetter();
1113             break;
1114         case GetSetter:
1115             compileGetSetter();
1116             break;
1117         case MultiGetByOffset:
1118             compileMultiGetByOffset();
1119             break;
1120         case PutByOffset:
1121             compilePutByOffset();
1122             break;
1123         case MultiPutByOffset:
1124             compileMultiPutByOffset();
1125             break;
1126         case MatchStructure:
1127             compileMatchStructure();
1128             break;
1129         case GetGlobalVar:
1130         case GetGlobalLexicalVariable:
1131             compileGetGlobalVariable();
1132             break;
1133         case PutGlobalVariable:
1134             compilePutGlobalVariable();
1135             break;
1136         case NotifyWrite:
1137             compileNotifyWrite();
1138             break;
1139         case GetCallee:
1140             compileGetCallee();
1141             break;
1142         case SetCallee:
1143             compileSetCallee();
1144             break;
1145         case GetArgumentCountIncludingThis:
1146             compileGetArgumentCountIncludingThis();
1147             break;
1148         case SetArgumentCountIncludingThis:
1149             compileSetArgumentCountIncludingThis();
1150             break;
1151         case GetScope:
1152             compileGetScope();
1153             break;
1154         case SkipScope:
1155             compileSkipScope();
1156             break;
1157         case GetGlobalObject:
1158             compileGetGlobalObject();
1159             break;
1160         case GetGlobalThis:
1161             compileGetGlobalThis();
1162             break;
1163         case GetClosureVar:
1164             compileGetClosureVar();
1165             break;
1166         case PutClosureVar:
1167             compilePutClosureVar();
1168             break;
1169         case GetFromArguments:
1170             compileGetFromArguments();
1171             break;
1172         case PutToArguments:
1173             compilePutToArguments();
1174             break;
1175         case GetArgument:
1176             compileGetArgument();
1177             break;
1178         case CompareEq:
1179             compileCompareEq();
1180             break;
1181         case CompareStrictEq:
1182             compileCompareStrictEq();
1183             break;
1184         case CompareLess:
1185             compileCompareLess();
1186             break;
1187         case CompareLessEq:
1188             compileCompareLessEq();
1189             break;
1190         case CompareGreater:
1191             compileCompareGreater();
1192             break;
1193         case CompareGreaterEq:
1194             compileCompareGreaterEq();
1195             break;
1196         case CompareBelow:
1197             compileCompareBelow();
1198             break;
1199         case CompareBelowEq:
1200             compileCompareBelowEq();
1201             break;
1202         case CompareEqPtr:
1203             compileCompareEqPtr();
1204             break;
1205         case SameValue:
1206             compileSameValue();
1207             break;
1208         case LogicalNot:
1209             compileLogicalNot();
1210             break;
1211         case Call:
1212         case TailCallInlinedCaller:
1213         case Construct:
1214             compileCallOrConstruct();
1215             break;
1216         case DirectCall:
1217         case DirectTailCallInlinedCaller:
1218         case DirectConstruct:
1219         case DirectTailCall:
1220             compileDirectCallOrConstruct();
1221             break;
1222         case TailCall:
1223             compileTailCall();
1224             break;
1225         case CallVarargs:
1226         case CallForwardVarargs:
1227         case TailCallVarargs:
1228         case TailCallVarargsInlinedCaller:
1229         case TailCallForwardVarargs:
1230         case TailCallForwardVarargsInlinedCaller:
1231         case ConstructVarargs:
1232         case ConstructForwardVarargs:
1233             compileCallOrConstructVarargs();
1234             break;
1235         case CallEval:
1236             compileCallEval();
1237             break;
1238         case LoadVarargs:
1239             compileLoadVarargs();
1240             break;
1241         case ForwardVarargs:
1242             compileForwardVarargs();
1243             break;
1244         case DFG::Jump:
1245             compileJump();
1246             break;
1247         case DFG::Branch:
1248             compileBranch();
1249             break;
1250         case DFG::Switch:
1251             compileSwitch();
1252             break;
1253         case DFG::EntrySwitch:
1254             compileEntrySwitch();
1255             break;
1256         case DFG::Return:
1257             compileReturn();
1258             break;
1259         case ForceOSRExit:
1260             compileForceOSRExit();
1261             break;
1262         case CPUIntrinsic:
1263 #if CPU(X86_64)
1264             compileCPUIntrinsic();
1265 #else
1266             RELEASE_ASSERT_NOT_REACHED();
1267 #endif
1268             break;
1269         case Throw:
1270             compileThrow();
1271             break;
1272         case ThrowStaticError:
1273             compileThrowStaticError();
1274             break;
1275         case InvalidationPoint:
1276             compileInvalidationPoint();
1277             break;
1278         case IsEmpty:
1279             compileIsEmpty();
1280             break;
1281         case IsUndefined:
1282             compileIsUndefined();
1283             break;
1284         case IsUndefinedOrNull:
1285             compileIsUndefinedOrNull();
1286             break;
1287         case IsBoolean:
1288             compileIsBoolean();
1289             break;
1290         case IsNumber:
1291             compileIsNumber();
1292             break;
1293         case NumberIsInteger:
1294             compileNumberIsInteger();
1295             break;
1296         case IsCellWithType:
1297             compileIsCellWithType();
1298             break;
1299         case MapHash:
1300             compileMapHash();
1301             break;
1302         case NormalizeMapKey:
1303             compileNormalizeMapKey();
1304             break;
1305         case GetMapBucket:
1306             compileGetMapBucket();
1307             break;
1308         case GetMapBucketHead:
1309             compileGetMapBucketHead();
1310             break;
1311         case GetMapBucketNext:
1312             compileGetMapBucketNext();
1313             break;
1314         case LoadKeyFromMapBucket:
1315             compileLoadKeyFromMapBucket();
1316             break;
1317         case LoadValueFromMapBucket:
1318             compileLoadValueFromMapBucket();
1319             break;
1320         case ExtractValueFromWeakMapGet:
1321             compileExtractValueFromWeakMapGet();
1322             break;
1323         case SetAdd:
1324             compileSetAdd();
1325             break;
1326         case MapSet:
1327             compileMapSet();
1328             break;
1329         case WeakMapGet:
1330             compileWeakMapGet();
1331             break;
1332         case WeakSetAdd:
1333             compileWeakSetAdd();
1334             break;
1335         case WeakMapSet:
1336             compileWeakMapSet();
1337             break;
1338         case IsObject:
1339             compileIsObject();
1340             break;
1341         case IsObjectOrNull:
1342             compileIsObjectOrNull();
1343             break;
1344         case IsFunction:
1345             compileIsFunction();
1346             break;
1347         case IsTypedArrayView:
1348             compileIsTypedArrayView();
1349             break;
1350         case ParseInt:
1351             compileParseInt();
1352             break;
1353         case TypeOf:
1354             compileTypeOf();
1355             break;
1356         case CheckTypeInfoFlags:
1357             compileCheckTypeInfoFlags();
1358             break;
1359         case OverridesHasInstance:
1360             compileOverridesHasInstance();
1361             break;
1362         case InstanceOf:
1363             compileInstanceOf();
1364             break;
1365         case InstanceOfCustom:
1366             compileInstanceOfCustom();
1367             break;
1368         case CountExecution:
1369             compileCountExecution();
1370             break;
1371         case SuperSamplerBegin:
1372             compileSuperSamplerBegin();
1373             break;
1374         case SuperSamplerEnd:
1375             compileSuperSamplerEnd();
1376             break;
1377         case StoreBarrier:
1378         case FencedStoreBarrier:
1379             compileStoreBarrier();
1380             break;
1381         case HasIndexedProperty:
1382             compileHasIndexedProperty();
1383             break;
1384         case HasGenericProperty:
1385             compileHasGenericProperty();
1386             break;
1387         case HasStructureProperty:
1388             compileHasStructureProperty();
1389             break;
1390         case GetDirectPname:
1391             compileGetDirectPname();
1392             break;
1393         case GetEnumerableLength:
1394             compileGetEnumerableLength();
1395             break;
1396         case GetPropertyEnumerator:
1397             compileGetPropertyEnumerator();
1398             break;
1399         case GetEnumeratorStructurePname:
1400             compileGetEnumeratorStructurePname();
1401             break;
1402         case GetEnumeratorGenericPname:
1403             compileGetEnumeratorGenericPname();
1404             break;
1405         case ToIndexString:
1406             compileToIndexString();
1407             break;
1408         case CheckStructureImmediate:
1409             compileCheckStructureImmediate();
1410             break;
1411         case MaterializeNewObject:
1412             compileMaterializeNewObject();
1413             break;
1414         case MaterializeCreateActivation:
1415             compileMaterializeCreateActivation();
1416             break;
1417         case CheckTraps:
1418             compileCheckTraps();
1419             break;
1420         case CreateRest:
1421             compileCreateRest();
1422             break;
1423         case GetRestLength:
1424             compileGetRestLength();
1425             break;
1426         case RegExpExec:
1427             compileRegExpExec();
1428             break;
1429         case RegExpExecNonGlobalOrSticky:
1430             compileRegExpExecNonGlobalOrSticky();
1431             break;
1432         case RegExpTest:
1433             compileRegExpTest();
1434             break;
1435         case RegExpMatchFast:
1436             compileRegExpMatchFast();
1437             break;
1438         case RegExpMatchFastGlobal:
1439             compileRegExpMatchFastGlobal();
1440             break;
1441         case NewRegexp:
1442             compileNewRegexp();
1443             break;
1444         case SetFunctionName:
1445             compileSetFunctionName();
1446             break;
1447         case StringReplace:
1448         case StringReplaceRegExp:
1449             compileStringReplace();
1450             break;
1451         case GetRegExpObjectLastIndex:
1452             compileGetRegExpObjectLastIndex();
1453             break;
1454         case SetRegExpObjectLastIndex:
1455             compileSetRegExpObjectLastIndex();
1456             break;
1457         case LogShadowChickenPrologue:
1458             compileLogShadowChickenPrologue();
1459             break;
1460         case LogShadowChickenTail:
1461             compileLogShadowChickenTail();
1462             break;
1463         case RecordRegExpCachedResult:
1464             compileRecordRegExpCachedResult();
1465             break;
1466         case ResolveScopeForHoistingFuncDeclInEval:
1467             compileResolveScopeForHoistingFuncDeclInEval();
1468             break;
1469         case ResolveScope:
1470             compileResolveScope();
1471             break;
1472         case GetDynamicVar:
1473             compileGetDynamicVar();
1474             break;
1475         case PutDynamicVar:
1476             compilePutDynamicVar();
1477             break;
1478         case Unreachable:
1479             compileUnreachable();
1480             break;
1481         case StringSlice:
1482             compileStringSlice();
1483             break;
1484         case ToLowerCase:
1485             compileToLowerCase();
1486             break;
1487         case NumberToStringWithRadix:
1488             compileNumberToStringWithRadix();
1489             break;
1490         case NumberToStringWithValidRadixConstant:
1491             compileNumberToStringWithValidRadixConstant();
1492             break;
1493         case CheckSubClass:
1494             compileCheckSubClass();
1495             break;
1496         case CallDOM:
1497             compileCallDOM();
1498             break;
1499         case CallDOMGetter:
1500             compileCallDOMGetter();
1501             break;
1502         case FilterCallLinkStatus:
1503         case FilterGetByIdStatus:
1504         case FilterPutByIdStatus:
1505         case FilterInByIdStatus:
1506             compileFilterICStatus();
1507             break;
1508         case DataViewGetInt:
1509         case DataViewGetFloat:
1510             compileDataViewGet();
1511             break;
1512         case DataViewSet:
1513             compileDataViewSet();
1514             break;
1515
1516         case PhantomLocal:
1517         case LoopHint:
1518         case MovHint:
1519         case ZombieHint:
1520         case ExitOK:
1521         case PhantomNewObject:
1522         case PhantomNewFunction:
1523         case PhantomNewGeneratorFunction:
1524         case PhantomNewAsyncGeneratorFunction:
1525         case PhantomNewAsyncFunction:
1526         case PhantomCreateActivation:
1527         case PhantomDirectArguments:
1528         case PhantomCreateRest:
1529         case PhantomSpread:
1530         case PhantomNewArrayWithSpread:
1531         case PhantomNewArrayBuffer:
1532         case PhantomClonedArguments:
1533         case PhantomNewRegexp:
1534         case PutHint:
1535         case BottomValue:
1536         case KillStack:
1537         case InitializeEntrypointArguments:
1538             break;
1539         default:
1540             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
1541             break;
1542         }
1543         
1544         if (m_node->isTerminal())
1545             return false;
1546         
1547         if (!m_state.isValid()) {
1548             safelyInvalidateAfterTermination();
1549             return false;
1550         }
1551
1552         m_availabilityCalculator.executeNode(m_node);
1553         m_interpreter.executeEffects(nodeIndex);
1554         
1555         return true;
1556     }
1557
1558     void compileUpsilon()
1559     {
1560         LValue upsilonValue = nullptr;
1561         switch (m_node->child1().useKind()) {
1562         case DoubleRepUse:
1563             upsilonValue = lowDouble(m_node->child1());
1564             break;
1565         case Int32Use:
1566         case KnownInt32Use:
1567             upsilonValue = lowInt32(m_node->child1());
1568             break;
1569         case Int52RepUse:
1570             upsilonValue = lowInt52(m_node->child1());
1571             break;
1572         case BooleanUse:
1573         case KnownBooleanUse:
1574             upsilonValue = lowBoolean(m_node->child1());
1575             break;
1576         case CellUse:
1577         case KnownCellUse:
1578             upsilonValue = lowCell(m_node->child1());
1579             break;
1580         case UntypedUse:
1581             upsilonValue = lowJSValue(m_node->child1());
1582             break;
1583         default:
1584             DFG_CRASH(m_graph, m_node, "Bad use kind");
1585             break;
1586         }
1587         ValueFromBlock upsilon = m_out.anchor(upsilonValue);
1588         LValue phiNode = m_phis.get(m_node->phi());
1589         m_out.addIncomingToPhi(phiNode, upsilon);
1590     }
1591     
1592     void compilePhi()
1593     {
1594         LValue phi = m_phis.get(m_node);
1595         m_out.m_block->append(phi);
1596
1597         switch (m_node->flags() & NodeResultMask) {
1598         case NodeResultDouble:
1599             setDouble(phi);
1600             break;
1601         case NodeResultInt32:
1602             setInt32(phi);
1603             break;
1604         case NodeResultInt52:
1605             setInt52(phi);
1606             break;
1607         case NodeResultBoolean:
1608             setBoolean(phi);
1609             break;
1610         case NodeResultJS:
1611             setJSValue(phi);
1612             break;
1613         default:
1614             DFG_CRASH(m_graph, m_node, "Bad result type");
1615             break;
1616         }
1617     }
1618     
1619     void compileDoubleConstant()
1620     {
1621         setDouble(m_out.constDouble(m_node->asNumber()));
1622     }
1623     
1624     void compileInt52Constant()
1625     {
1626         int64_t value = m_node->asAnyInt();
1627         
1628         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
1629         setStrictInt52(m_out.constInt64(value));
1630     }
1631
1632     void compileLazyJSConstant()
1633     {
1634         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
1635         LazyJSValue value = m_node->lazyJSValue();
1636         patchpoint->setGenerator(
1637             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1638                 value.emit(jit, JSValueRegs(params[0].gpr()));
1639             });
1640         patchpoint->effects = Effects::none();
1641         setJSValue(patchpoint);
1642     }
1643
1644     void compileDoubleRep()
1645     {
1646         switch (m_node->child1().useKind()) {
1647         case RealNumberUse: {
1648             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1649             
1650             LValue doubleValue = unboxDouble(value);
1651             
1652             LBasicBlock intCase = m_out.newBlock();
1653             LBasicBlock continuation = m_out.newBlock();
1654             
1655             ValueFromBlock fastResult = m_out.anchor(doubleValue);
1656             m_out.branch(
1657                 m_out.doubleEqual(doubleValue, doubleValue),
1658                 usually(continuation), rarely(intCase));
1659             
1660             LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
1661
1662             FTL_TYPE_CHECK(
1663                 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
1664                 isNotInt32(value, provenType(m_node->child1()) & ~SpecDoubleReal));
1665             ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
1666             m_out.jump(continuation);
1667             
1668             m_out.appendTo(continuation, lastNext);
1669             
1670             setDouble(m_out.phi(Double, fastResult, slowResult));
1671             return;
1672         }
1673             
1674         case NotCellUse:
1675         case NumberUse: {
1676             bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
1677             
1678             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1679
1680             LBasicBlock intCase = m_out.newBlock();
1681             LBasicBlock doubleTesting = m_out.newBlock();
1682             LBasicBlock doubleCase = m_out.newBlock();
1683             LBasicBlock nonDoubleCase = m_out.newBlock();
1684             LBasicBlock continuation = m_out.newBlock();
1685             
1686             m_out.branch(
1687                 isNotInt32(value, provenType(m_node->child1())),
1688                 unsure(doubleTesting), unsure(intCase));
1689             
1690             LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1691             
1692             ValueFromBlock intToDouble = m_out.anchor(
1693                 m_out.intToDouble(unboxInt32(value)));
1694             m_out.jump(continuation);
1695             
1696             m_out.appendTo(doubleTesting, doubleCase);
1697             LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1698             m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1699
1700             m_out.appendTo(doubleCase, nonDoubleCase);
1701             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1702             m_out.jump(continuation);
1703
1704             if (shouldConvertNonNumber) {
1705                 LBasicBlock undefinedCase = m_out.newBlock();
1706                 LBasicBlock testNullCase = m_out.newBlock();
1707                 LBasicBlock nullCase = m_out.newBlock();
1708                 LBasicBlock testBooleanTrueCase = m_out.newBlock();
1709                 LBasicBlock convertBooleanTrueCase = m_out.newBlock();
1710                 LBasicBlock convertBooleanFalseCase = m_out.newBlock();
1711
1712                 m_out.appendTo(nonDoubleCase, undefinedCase);
1713                 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1714                 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1715
1716                 m_out.appendTo(undefinedCase, testNullCase);
1717                 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1718                 m_out.jump(continuation);
1719
1720                 m_out.appendTo(testNullCase, nullCase);
1721                 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1722                 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1723
1724                 m_out.appendTo(nullCase, testBooleanTrueCase);
1725                 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1726                 m_out.jump(continuation);
1727
1728                 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1729                 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1730                 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1731
1732                 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1733                 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1734                 m_out.jump(continuation);
1735
1736                 m_out.appendTo(convertBooleanFalseCase, continuation);
1737
1738                 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1739                 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCellCheck, valueIsNotBooleanFalse);
1740                 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1741                 m_out.jump(continuation);
1742
1743                 m_out.appendTo(continuation, lastNext);
1744                 setDouble(m_out.phi(Double, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1745                 return;
1746             }
1747             m_out.appendTo(nonDoubleCase, continuation);
1748             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1749             m_out.unreachable();
1750
1751             m_out.appendTo(continuation, lastNext);
1752
1753             setDouble(m_out.phi(Double, intToDouble, unboxedDouble));
1754             return;
1755         }
1756             
1757         case Int52RepUse: {
1758             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1759             return;
1760         }
1761             
1762         default:
1763             DFG_CRASH(m_graph, m_node, "Bad use kind");
1764         }
1765     }
1766
1767     void compileDoubleAsInt32()
1768     {
1769         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1770         setInt32(integerValue);
1771     }
1772
1773     void compileValueRep()
1774     {
1775         switch (m_node->child1().useKind()) {
1776         case DoubleRepUse: {
1777             LValue value = lowDouble(m_node->child1());
1778             
1779             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1780                 value = m_out.select(
1781                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1782             }
1783             
1784             setJSValue(boxDouble(value));
1785             return;
1786         }
1787             
1788         case Int52RepUse: {
1789             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1790             return;
1791         }
1792             
1793         default:
1794             DFG_CRASH(m_graph, m_node, "Bad use kind");
1795         }
1796     }
1797     
1798     void compileInt52Rep()
1799     {
1800         switch (m_node->child1().useKind()) {
1801         case Int32Use:
1802             setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1())));
1803             return;
1804             
1805         case AnyIntUse:
1806             setStrictInt52(
1807                 jsValueToStrictInt52(
1808                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1809             return;
1810             
1811         case DoubleRepAnyIntUse:
1812             setStrictInt52(
1813                 doubleToStrictInt52(
1814                     m_node->child1(), lowDouble(m_node->child1())));
1815             return;
1816             
1817         default:
1818             RELEASE_ASSERT_NOT_REACHED();
1819         }
1820     }
1821     
1822     void compileValueToInt32()
1823     {
1824         switch (m_node->child1().useKind()) {
1825         case Int52RepUse:
1826             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1827             break;
1828             
1829         case DoubleRepUse:
1830             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1831             break;
1832             
1833         case NumberUse:
1834         case NotCellUse: {
1835             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1836             if (isValid(value)) {
1837                 setInt32(value.value());
1838                 break;
1839             }
1840             
1841             value = m_jsValueValues.get(m_node->child1().node());
1842             if (isValid(value)) {
1843                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1844                 break;
1845             }
1846             
1847             // We'll basically just get here for constants. But it's good to have this
1848             // catch-all since we often add new representations into the mix.
1849             setInt32(
1850                 numberOrNotCellToInt32(
1851                     m_node->child1(),
1852                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1853             break;
1854         }
1855             
1856         default:
1857             DFG_CRASH(m_graph, m_node, "Bad use kind");
1858             break;
1859         }
1860     }
1861     
1862     void compileBooleanToNumber()
1863     {
1864         switch (m_node->child1().useKind()) {
1865         case BooleanUse: {
1866             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), Int32));
1867             return;
1868         }
1869             
1870         case UntypedUse: {
1871             LValue value = lowJSValue(m_node->child1());
1872             
1873             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1874                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1875                 return;
1876             }
1877             
1878             LBasicBlock booleanCase = m_out.newBlock();
1879             LBasicBlock continuation = m_out.newBlock();
1880             
1881             ValueFromBlock notBooleanResult = m_out.anchor(value);
1882             m_out.branch(
1883                 isBoolean(value, provenType(m_node->child1())),
1884                 unsure(booleanCase), unsure(continuation));
1885             
1886             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1887             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1888                 m_out.zeroExt(unboxBoolean(value), Int64), m_tagTypeNumber));
1889             m_out.jump(continuation);
1890             
1891             m_out.appendTo(continuation, lastNext);
1892             setJSValue(m_out.phi(Int64, booleanResult, notBooleanResult));
1893             return;
1894         }
1895             
1896         default:
1897             RELEASE_ASSERT_NOT_REACHED();
1898             return;
1899         }
1900     }
1901
1902     void compileExtractOSREntryLocal()
1903     {
1904         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1905             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1906         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1907     }
1908
1909     void compileExtractCatchLocal()
1910     {
1911         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(m_ftlState.jitCode->common.catchOSREntryBuffer->dataBuffer());
1912         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->catchOSREntryIndex())));
1913     }
1914
1915     void compileClearCatchLocals()
1916     {
1917         ScratchBuffer* scratchBuffer = m_ftlState.jitCode->common.catchOSREntryBuffer;
1918         ASSERT(scratchBuffer);
1919         m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
1920     }
1921     
1922     void compileGetStack()
1923     {
1924         StackAccessData* data = m_node->stackAccessData();
1925         AbstractValue& value = m_state.operand(data->local);
1926         
1927         DFG_ASSERT(m_graph, m_node, isConcrete(data->format), data->format);
1928         
1929         switch (data->format) {
1930         case FlushedDouble:
1931             setDouble(m_out.loadDouble(addressFor(data->machineLocal)));
1932             break;
1933         case FlushedInt52:
1934             setInt52(m_out.load64(addressFor(data->machineLocal)));
1935             break;
1936         default:
1937             if (isInt32Speculation(value.m_type))
1938                 setInt32(m_out.load32(payloadFor(data->machineLocal)));
1939             else
1940                 setJSValue(m_out.load64(addressFor(data->machineLocal)));
1941             break;
1942         }
1943     }
1944     
1945     void compilePutStack()
1946     {
1947         StackAccessData* data = m_node->stackAccessData();
1948         switch (data->format) {
1949         case FlushedJSValue: {
1950             LValue value = lowJSValue(m_node->child1());
1951             m_out.store64(value, addressFor(data->machineLocal));
1952             break;
1953         }
1954             
1955         case FlushedDouble: {
1956             LValue value = lowDouble(m_node->child1());
1957             m_out.storeDouble(value, addressFor(data->machineLocal));
1958             break;
1959         }
1960             
1961         case FlushedInt32: {
1962             LValue value = lowInt32(m_node->child1());
1963             m_out.store32(value, payloadFor(data->machineLocal));
1964             break;
1965         }
1966             
1967         case FlushedInt52: {
1968             LValue value = lowInt52(m_node->child1());
1969             m_out.store64(value, addressFor(data->machineLocal));
1970             break;
1971         }
1972             
1973         case FlushedCell: {
1974             LValue value = lowCell(m_node->child1());
1975             m_out.store64(value, addressFor(data->machineLocal));
1976             break;
1977         }
1978             
1979         case FlushedBoolean: {
1980             speculateBoolean(m_node->child1());
1981             m_out.store64(
1982                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1983                 addressFor(data->machineLocal));
1984             break;
1985         }
1986             
1987         default:
1988             DFG_CRASH(m_graph, m_node, "Bad flush format");
1989             break;
1990         }
1991     }
1992     
1993     void compileNoOp()
1994     {
1995         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1996     }
1997
1998     void compileToObjectOrCallObjectConstructor()
1999     {
2000         LValue value = lowJSValue(m_node->child1());
2001
2002         LBasicBlock isCellCase = m_out.newBlock();
2003         LBasicBlock slowCase = m_out.newBlock();
2004         LBasicBlock continuation = m_out.newBlock();
2005
2006         m_out.branch(isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2007
2008         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2009         ValueFromBlock fastResult = m_out.anchor(value);
2010         m_out.branch(isObject(value), usually(continuation), rarely(slowCase));
2011
2012         m_out.appendTo(slowCase, continuation);
2013
2014         ValueFromBlock slowResult;
2015         if (m_node->op() == ToObject) {
2016             auto* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2017             slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationToObject), m_callFrame, weakPointer(globalObject), value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2018         } else
2019             slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationCallObjectConstructor), m_callFrame, frozenPointer(m_node->cellOperand()), value));
2020         m_out.jump(continuation);
2021
2022         m_out.appendTo(continuation, lastNext);
2023         setJSValue(m_out.phi(Int64, fastResult, slowResult));
2024     }
2025     
2026     void compileToThis()
2027     {
2028         LValue value = lowJSValue(m_node->child1());
2029         
2030         LBasicBlock isCellCase = m_out.newBlock();
2031         LBasicBlock slowCase = m_out.newBlock();
2032         LBasicBlock continuation = m_out.newBlock();
2033         
2034         m_out.branch(
2035             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2036         
2037         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2038         ValueFromBlock fastResult = m_out.anchor(value);
2039         m_out.branch(
2040             m_out.testIsZero32(
2041                 m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags),
2042                 m_out.constInt32(OverridesToThis)),
2043             usually(continuation), rarely(slowCase));
2044         
2045         m_out.appendTo(slowCase, continuation);
2046         J_JITOperation_EJ function;
2047         if (m_graph.isStrictModeFor(m_node->origin.semantic))
2048             function = operationToThisStrict;
2049         else
2050             function = operationToThis;
2051         ValueFromBlock slowResult = m_out.anchor(
2052             vmCall(Int64, m_out.operation(function), m_callFrame, value));
2053         m_out.jump(continuation);
2054         
2055         m_out.appendTo(continuation, lastNext);
2056         setJSValue(m_out.phi(Int64, fastResult, slowResult));
2057     }
2058
2059     void compileValueAdd()
2060     {
2061         if (m_node->isBinaryUseKind(BigIntUse)) {
2062             LValue left = lowBigInt(m_node->child1());
2063             LValue right = lowBigInt(m_node->child2());
2064
2065             LValue result = vmCall(pointerType(), m_out.operation(operationAddBigInt), m_callFrame, left, right);
2066             setJSValue(result);
2067             return;
2068         }
2069
2070         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2071         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2072         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2073         auto repatchingFunction = operationValueAddOptimize;
2074         auto nonRepatchingFunction = operationValueAdd;
2075         compileBinaryMathIC<JITAddGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2076     }
2077
2078     void compileValueSub()
2079     {
2080         if (m_node->isBinaryUseKind(BigIntUse)) {
2081             LValue left = lowBigInt(m_node->child1());
2082             LValue right = lowBigInt(m_node->child2());
2083             
2084             LValue result = vmCall(pointerType(), m_out.operation(operationSubBigInt), m_callFrame, left, right);
2085             setJSValue(result);
2086             return;
2087         }
2088
2089         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2090         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2091         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2092         auto repatchingFunction = operationValueSubOptimize;
2093         auto nonRepatchingFunction = operationValueSub;
2094         compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2095     }
2096
2097     void compileValueMul()
2098     {
2099         if (m_node->isBinaryUseKind(BigIntUse)) {
2100             LValue left = lowBigInt(m_node->child1());
2101             LValue right = lowBigInt(m_node->child2());
2102             
2103             LValue result = vmCall(Int64, m_out.operation(operationMulBigInt), m_callFrame, left, right);
2104             setJSValue(result);
2105             return;
2106         }
2107
2108         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2109         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2110         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2111         auto repatchingFunction = operationValueMulOptimize;
2112         auto nonRepatchingFunction = operationValueMul;
2113         compileBinaryMathIC<JITMulGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2114     }
2115
2116     template <typename Generator, typename Func1, typename Func2,
2117         typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2118     void compileUnaryMathIC(ArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2119     {
2120         Node* node = m_node;
2121
2122         LValue operand = lowJSValue(node->child1());
2123
2124         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2125         patchpoint->appendSomeRegister(operand);
2126         patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister));
2127         patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister));
2128         RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
2129         patchpoint->numGPScratchRegisters = 1;
2130         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2131         State* state = &m_ftlState;
2132         patchpoint->setGenerator(
2133             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2134                 AllowMacroScratchRegisterUsage allowScratch(jit);
2135
2136                 Box<CCallHelpers::JumpList> exceptions =
2137                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2138
2139 #if ENABLE(MATH_IC_STATS)
2140                 auto inlineStart = jit.label();
2141 #endif
2142
2143                 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2144                 JITUnaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2145                 mathIC->m_generator = Generator(JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), params.gpScratch(0));
2146
2147                 bool shouldEmitProfiling = false;
2148                 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2149
2150                 if (generatedInline) {
2151                     ASSERT(!mathICGenerationState->slowPathJumps.empty());
2152                     auto done = jit.label();
2153                     params.addLatePath([=] (CCallHelpers& jit) {
2154                         AllowMacroScratchRegisterUsage allowScratch(jit);
2155                         mathICGenerationState->slowPathJumps.link(&jit);
2156                         mathICGenerationState->slowPathStart = jit.label();
2157 #if ENABLE(MATH_IC_STATS)
2158                         auto slowPathStart = jit.label();
2159 #endif
2160
2161                         if (mathICGenerationState->shouldSlowPathRepatch) {
2162                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2163                                 repatchingFunction, params[0].gpr(), params[1].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2164                             mathICGenerationState->slowPathCall = call.call();
2165                         } else {
2166                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2167                                 exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr());
2168                             mathICGenerationState->slowPathCall = call.call();
2169                         }
2170                         jit.jump().linkTo(done, &jit);
2171
2172                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2173                             mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2174                         });
2175
2176 #if ENABLE(MATH_IC_STATS)
2177                         auto slowPathEnd = jit.label();
2178                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2179                             size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2180                             mathIC->m_generatedCodeSize += size;
2181                         });
2182 #endif
2183                     });
2184                 } else {
2185                     callOperation(
2186                         *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2187                         nonRepatchingFunction, params[0].gpr(), params[1].gpr());
2188                 }
2189
2190 #if ENABLE(MATH_IC_STATS)
2191                 auto inlineEnd = jit.label();
2192                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2193                     size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2194                     mathIC->m_generatedCodeSize += size;
2195                 });
2196 #endif
2197             });
2198
2199         setJSValue(patchpoint);
2200     }
2201
2202     template <typename Generator, typename Func1, typename Func2,
2203         typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2204     void compileBinaryMathIC(ArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2205     {
2206         Node* node = m_node;
2207         
2208         LValue left = lowJSValue(node->child1());
2209         LValue right = lowJSValue(node->child2());
2210
2211         SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
2212         SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
2213             
2214         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2215         patchpoint->appendSomeRegister(left);
2216         patchpoint->appendSomeRegister(right);
2217         patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister));
2218         patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister));
2219         RefPtr<PatchpointExceptionHandle> exceptionHandle =
2220             preparePatchpointForExceptions(patchpoint);
2221         patchpoint->numGPScratchRegisters = 1;
2222         patchpoint->numFPScratchRegisters = 2;
2223         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2224         State* state = &m_ftlState;
2225         patchpoint->setGenerator(
2226             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2227                 AllowMacroScratchRegisterUsage allowScratch(jit);
2228
2229
2230                 Box<CCallHelpers::JumpList> exceptions =
2231                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2232
2233 #if ENABLE(MATH_IC_STATS)
2234                 auto inlineStart = jit.label();
2235 #endif
2236
2237                 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2238                 JITBinaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2239                 mathIC->m_generator = Generator(leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
2240                     JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0),
2241                     params.fpScratch(1), params.gpScratch(0), InvalidFPRReg);
2242
2243                 bool shouldEmitProfiling = false;
2244                 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2245
2246                 if (generatedInline) {
2247                     ASSERT(!mathICGenerationState->slowPathJumps.empty());
2248                     auto done = jit.label();
2249                     params.addLatePath([=] (CCallHelpers& jit) {
2250                         AllowMacroScratchRegisterUsage allowScratch(jit);
2251                         mathICGenerationState->slowPathJumps.link(&jit);
2252                         mathICGenerationState->slowPathStart = jit.label();
2253 #if ENABLE(MATH_IC_STATS)
2254                         auto slowPathStart = jit.label();
2255 #endif
2256
2257                         if (mathICGenerationState->shouldSlowPathRepatch) {
2258                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2259                                 repatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2260                             mathICGenerationState->slowPathCall = call.call();
2261                         } else {
2262                             SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2263                                 exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr());
2264                             mathICGenerationState->slowPathCall = call.call();
2265                         }
2266                         jit.jump().linkTo(done, &jit);
2267
2268                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2269                             mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2270                         });
2271
2272 #if ENABLE(MATH_IC_STATS)
2273                         auto slowPathEnd = jit.label();
2274                         jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2275                             size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2276                             mathIC->m_generatedCodeSize += size;
2277                         });
2278 #endif
2279                     });
2280                 } else {
2281                     callOperation(
2282                         *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2283                         nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr());
2284                 }
2285
2286 #if ENABLE(MATH_IC_STATS)
2287                 auto inlineEnd = jit.label();
2288                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2289                     size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2290                     mathIC->m_generatedCodeSize += size;
2291                 });
2292 #endif
2293             });
2294
2295         setJSValue(patchpoint);
2296     }
2297     
2298     void compileStrCat()
2299     {
2300         LValue result;
2301         if (m_node->child3()) {
2302             result = vmCall(
2303                 Int64, m_out.operation(operationStrCat3), m_callFrame,
2304                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2305                 lowJSValue(m_node->child2(), ManualOperandSpeculation),
2306                 lowJSValue(m_node->child3(), ManualOperandSpeculation));
2307         } else {
2308             result = vmCall(
2309                 Int64, m_out.operation(operationStrCat2), m_callFrame,
2310                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2311                 lowJSValue(m_node->child2(), ManualOperandSpeculation));
2312         }
2313         setJSValue(result);
2314     }
2315     
2316     void compileArithAddOrSub()
2317     {
2318         bool isSub =  m_node->op() == ArithSub;
2319         switch (m_node->binaryUseKind()) {
2320         case Int32Use: {
2321             LValue left = lowInt32(m_node->child1());
2322             LValue right = lowInt32(m_node->child2());
2323
2324             if (!shouldCheckOverflow(m_node->arithMode())) {
2325                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
2326                 break;
2327             }
2328
2329             CheckValue* result =
2330                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2331             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2332             setInt32(result);
2333             break;
2334         }
2335             
2336         case Int52RepUse: {
2337             if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)
2338                 && !abstractValue(m_node->child2()).couldBeType(SpecNonInt32AsInt52)) {
2339                 Int52Kind kind;
2340                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
2341                 LValue right = lowInt52(m_node->child2(), kind);
2342                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
2343                 break;
2344             }
2345
2346             LValue left = lowInt52(m_node->child1());
2347             LValue right = lowInt52(m_node->child2());
2348             CheckValue* result =
2349                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2350             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2351             setInt52(result);
2352             break;
2353         }
2354             
2355         case DoubleRepUse: {
2356             LValue C1 = lowDouble(m_node->child1());
2357             LValue C2 = lowDouble(m_node->child2());
2358
2359             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
2360             break;
2361         }
2362
2363         case UntypedUse: {
2364             if (!isSub) {
2365                 DFG_CRASH(m_graph, m_node, "Bad use kind");
2366                 break;
2367             }
2368
2369             CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2370             unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2371             ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
2372             auto repatchingFunction = operationValueSubOptimize;
2373             auto nonRepatchingFunction = operationValueSub;
2374             compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2375             break;
2376         }
2377
2378         default:
2379             DFG_CRASH(m_graph, m_node, "Bad use kind");
2380             break;
2381         }
2382     }
2383
2384     void compileArithClz32()
2385     {
2386         if (m_node->child1().useKind() == Int32Use || m_node->child1().useKind() == KnownInt32Use) {
2387             LValue operand = lowInt32(m_node->child1());
2388             setInt32(m_out.ctlz32(operand));
2389             return;
2390         }
2391         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2392         LValue argument = lowJSValue(m_node->child1());
2393         LValue result = vmCall(Int32, m_out.operation(operationArithClz32), m_callFrame, argument);
2394         setInt32(result);
2395     }
2396     
2397     void compileArithMul()
2398     {
2399         switch (m_node->binaryUseKind()) {
2400         case Int32Use: {
2401             LValue left = lowInt32(m_node->child1());
2402             LValue right = lowInt32(m_node->child2());
2403             
2404             LValue result;
2405
2406             if (!shouldCheckOverflow(m_node->arithMode()))
2407                 result = m_out.mul(left, right);
2408             else {
2409                 CheckValue* speculation = m_out.speculateMul(left, right);
2410                 blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin);
2411                 result = speculation;
2412             }
2413             
2414             if (shouldCheckNegativeZero(m_node->arithMode())) {
2415                 LBasicBlock slowCase = m_out.newBlock();
2416                 LBasicBlock continuation = m_out.newBlock();
2417                 
2418                 m_out.branch(
2419                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
2420                 
2421                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2422                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero));
2423                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero));
2424                 m_out.jump(continuation);
2425                 m_out.appendTo(continuation, lastNext);
2426             }
2427             
2428             setInt32(result);
2429             break;
2430         }
2431             
2432         case Int52RepUse: {
2433             Int52Kind kind;
2434             LValue left = lowWhicheverInt52(m_node->child1(), kind);
2435             LValue right = lowInt52(m_node->child2(), opposite(kind));
2436
2437             CheckValue* result = m_out.speculateMul(left, right);
2438             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2439
2440             if (shouldCheckNegativeZero(m_node->arithMode())) {
2441                 LBasicBlock slowCase = m_out.newBlock();
2442                 LBasicBlock continuation = m_out.newBlock();
2443                 
2444                 m_out.branch(
2445                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2446                 
2447                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2448                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero));
2449                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero));
2450                 m_out.jump(continuation);
2451                 m_out.appendTo(continuation, lastNext);
2452             }
2453             
2454             setInt52(result);
2455             break;
2456         }
2457             
2458         case DoubleRepUse: {
2459             setDouble(
2460                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2461             break;
2462         }
2463
2464         default:
2465             DFG_CRASH(m_graph, m_node, "Bad use kind");
2466             break;
2467         }
2468     }
2469
2470     void compileValueDiv()
2471     {
2472         if (m_node->isBinaryUseKind(BigIntUse)) {
2473             LValue left = lowBigInt(m_node->child1());
2474             LValue right = lowBigInt(m_node->child2());
2475             
2476             LValue result = vmCall(pointerType(), m_out.operation(operationDivBigInt), m_callFrame, left, right);
2477             setJSValue(result);
2478             return;
2479         }
2480
2481         emitBinarySnippet<JITDivGenerator, NeedScratchFPR>(operationValueDiv);
2482     }
2483
2484     void compileArithDiv()
2485     {
2486         switch (m_node->binaryUseKind()) {
2487         case Int32Use: {
2488             LValue numerator = lowInt32(m_node->child1());
2489             LValue denominator = lowInt32(m_node->child2());
2490
2491             if (shouldCheckNegativeZero(m_node->arithMode())) {
2492                 LBasicBlock zeroNumerator = m_out.newBlock();
2493                 LBasicBlock numeratorContinuation = m_out.newBlock();
2494
2495                 m_out.branch(
2496                     m_out.isZero32(numerator),
2497                     rarely(zeroNumerator), usually(numeratorContinuation));
2498
2499                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
2500
2501                 speculate(
2502                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
2503
2504                 m_out.jump(numeratorContinuation);
2505
2506                 m_out.appendTo(numeratorContinuation, innerLastNext);
2507             }
2508             
2509             if (shouldCheckOverflow(m_node->arithMode())) {
2510                 LBasicBlock unsafeDenominator = m_out.newBlock();
2511                 LBasicBlock continuation = m_out.newBlock();
2512
2513                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2514                 m_out.branch(
2515                     m_out.above(adjustedDenominator, m_out.int32One),
2516                     usually(continuation), rarely(unsafeDenominator));
2517
2518                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2519                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2520                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2521                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2522                 m_out.jump(continuation);
2523
2524                 m_out.appendTo(continuation, lastNext);
2525                 LValue result = m_out.div(numerator, denominator);
2526                 speculate(
2527                     Overflow, noValue(), 0,
2528                     m_out.notEqual(m_out.mul(result, denominator), numerator));
2529                 setInt32(result);
2530             } else
2531                 setInt32(m_out.chillDiv(numerator, denominator));
2532
2533             break;
2534         }
2535             
2536         case DoubleRepUse: {
2537             setDouble(m_out.doubleDiv(
2538                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2539             break;
2540         }
2541
2542         default:
2543             DFG_CRASH(m_graph, m_node, "Bad use kind");
2544             break;
2545         }
2546     }
2547     
2548     void compileValueMod()
2549     {
2550         if (m_node->binaryUseKind() == BigIntUse) {
2551             LValue left = lowBigInt(m_node->child1());
2552             LValue right = lowBigInt(m_node->child2());
2553
2554             LValue result = vmCall(pointerType(), m_out.operation(operationModBigInt), m_callFrame, left, right);
2555             setJSValue(result);
2556             return;
2557         }
2558
2559         DFG_ASSERT(m_graph, m_node, m_node->binaryUseKind() == UntypedUse, m_node->binaryUseKind());
2560         LValue left = lowJSValue(m_node->child1());
2561         LValue right = lowJSValue(m_node->child2());
2562         LValue result = vmCall(Int64, m_out.operation(operationValueMod), m_callFrame, left, right);
2563         setJSValue(result);
2564     }
2565
2566     void compileArithMod()
2567     {
2568         switch (m_node->binaryUseKind()) {
2569         case Int32Use: {
2570             LValue numerator = lowInt32(m_node->child1());
2571             LValue denominator = lowInt32(m_node->child2());
2572
2573             LValue remainder;
2574             if (shouldCheckOverflow(m_node->arithMode())) {
2575                 LBasicBlock unsafeDenominator = m_out.newBlock();
2576                 LBasicBlock continuation = m_out.newBlock();
2577
2578                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2579                 m_out.branch(
2580                     m_out.above(adjustedDenominator, m_out.int32One),
2581                     usually(continuation), rarely(unsafeDenominator));
2582
2583                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2584                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2585                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2586                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2587                 m_out.jump(continuation);
2588
2589                 m_out.appendTo(continuation, lastNext);
2590                 LValue result = m_out.mod(numerator, denominator);
2591                 remainder = result;
2592             } else
2593                 remainder = m_out.chillMod(numerator, denominator);
2594
2595             if (shouldCheckNegativeZero(m_node->arithMode())) {
2596                 LBasicBlock negativeNumerator = m_out.newBlock();
2597                 LBasicBlock numeratorContinuation = m_out.newBlock();
2598
2599                 m_out.branch(
2600                     m_out.lessThan(numerator, m_out.int32Zero),
2601                     unsure(negativeNumerator), unsure(numeratorContinuation));
2602
2603                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
2604
2605                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
2606
2607                 m_out.jump(numeratorContinuation);
2608
2609                 m_out.appendTo(numeratorContinuation, innerLastNext);
2610             }
2611
2612             setInt32(remainder);
2613             break;
2614         }
2615             
2616         case DoubleRepUse: {
2617             setDouble(
2618                 m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2619             break;
2620         }
2621             
2622         default:
2623             DFG_CRASH(m_graph, m_node, "Bad use kind");
2624             break;
2625         }
2626     }
2627
2628     void compileArithMinOrMax()
2629     {
2630         switch (m_node->binaryUseKind()) {
2631         case Int32Use: {
2632             LValue left = lowInt32(m_node->child1());
2633             LValue right = lowInt32(m_node->child2());
2634             
2635             setInt32(
2636                 m_out.select(
2637                     m_node->op() == ArithMin
2638                         ? m_out.lessThan(left, right)
2639                         : m_out.lessThan(right, left),
2640                     left, right));
2641             break;
2642         }
2643             
2644         case DoubleRepUse: {
2645             LValue left = lowDouble(m_node->child1());
2646             LValue right = lowDouble(m_node->child2());
2647             
2648             LBasicBlock notLessThan = m_out.newBlock();
2649             LBasicBlock continuation = m_out.newBlock();
2650             
2651             Vector<ValueFromBlock, 2> results;
2652             
2653             results.append(m_out.anchor(left));
2654             m_out.branch(
2655                 m_node->op() == ArithMin
2656                     ? m_out.doubleLessThan(left, right)
2657                     : m_out.doubleGreaterThan(left, right),
2658                 unsure(continuation), unsure(notLessThan));
2659             
2660             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
2661             results.append(m_out.anchor(m_out.select(
2662                 m_node->op() == ArithMin
2663                     ? m_out.doubleGreaterThanOrEqual(left, right)
2664                     : m_out.doubleLessThanOrEqual(left, right),
2665                 right, m_out.constDouble(PNaN))));
2666             m_out.jump(continuation);
2667             
2668             m_out.appendTo(continuation, lastNext);
2669             setDouble(m_out.phi(Double, results));
2670             break;
2671         }
2672             
2673         default:
2674             DFG_CRASH(m_graph, m_node, "Bad use kind");
2675             break;
2676         }
2677     }
2678     
2679     void compileArithAbs()
2680     {
2681         switch (m_node->child1().useKind()) {
2682         case Int32Use: {
2683             LValue value = lowInt32(m_node->child1());
2684
2685             LValue mask = m_out.aShr(value, m_out.constInt32(31));
2686             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
2687
2688             if (shouldCheckOverflow(m_node->arithMode()))
2689                 speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2690
2691             setInt32(result);
2692             break;
2693         }
2694             
2695         case DoubleRepUse: {
2696             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
2697             break;
2698         }
2699             
2700         default: {
2701             DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2702             LValue argument = lowJSValue(m_node->child1());
2703             LValue result = vmCall(Double, m_out.operation(operationArithAbs), m_callFrame, argument);
2704             setDouble(result);
2705             break;
2706         }
2707         }
2708     }
2709
2710     void compileArithUnary()
2711     {
2712         if (m_node->child1().useKind() == DoubleRepUse) {
2713             setDouble(m_out.doubleUnary(m_node->arithUnaryType(), lowDouble(m_node->child1())));
2714             return;
2715         }
2716         LValue argument = lowJSValue(m_node->child1());
2717         LValue result = vmCall(Double, m_out.operation(DFG::arithUnaryOperation(m_node->arithUnaryType())), m_callFrame, argument);
2718         setDouble(result);
2719     }
2720
2721     void compileArithPow()
2722     {
2723         if (m_node->child2().useKind() == Int32Use)
2724             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
2725         else {
2726             LValue base = lowDouble(m_node->child1());
2727             LValue exponent = lowDouble(m_node->child2());
2728
2729             LBasicBlock integerExponentIsSmallBlock = m_out.newBlock();
2730             LBasicBlock integerExponentPowBlock = m_out.newBlock();
2731             LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock();
2732             LBasicBlock nanExceptionBaseIsOne = m_out.newBlock();
2733             LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock();
2734             LBasicBlock testExponentIsOneHalf = m_out.newBlock();
2735             LBasicBlock handleBaseZeroExponentIsOneHalf = m_out.newBlock();
2736             LBasicBlock handleInfinityForExponentIsOneHalf = m_out.newBlock();
2737             LBasicBlock exponentIsOneHalfNormal = m_out.newBlock();
2738             LBasicBlock exponentIsOneHalfInfinity = m_out.newBlock();
2739             LBasicBlock testExponentIsNegativeOneHalf = m_out.newBlock();
2740             LBasicBlock testBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2741             LBasicBlock handleBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2742             LBasicBlock handleInfinityForExponentIsNegativeOneHalf = m_out.newBlock();
2743             LBasicBlock exponentIsNegativeOneHalfNormal = m_out.newBlock();
2744             LBasicBlock exponentIsNegativeOneHalfInfinity = m_out.newBlock();
2745             LBasicBlock powBlock = m_out.newBlock();
2746             LBasicBlock nanExceptionResultIsNaN = m_out.newBlock();
2747             LBasicBlock continuation = m_out.newBlock();
2748
2749             LValue integerExponent = m_out.doubleToInt(exponent);
2750             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
2751             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
2752             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
2753
2754             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
2755             LValue integerExponentBelowMax = m_out.belowOrEqual(integerExponent, m_out.constInt32(maxExponentForIntegerMathPow));
2756             m_out.branch(integerExponentBelowMax, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
2757
2758             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
2759             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
2760             m_out.jump(continuation);
2761
2762             // If y is NaN, the result is NaN.
2763             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionBaseIsOne);
2764             LValue exponentIsNaN;
2765             if (provenType(m_node->child2()) & SpecDoubleNaN)
2766                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
2767             else
2768                 exponentIsNaN = m_out.booleanFalse;
2769             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionBaseIsOne));
2770
2771             // If abs(x) is 1 and y is +infinity, the result is NaN.
2772             // If abs(x) is 1 and y is -infinity, the result is NaN.
2773
2774             //     Test if base == 1.
2775             m_out.appendTo(nanExceptionBaseIsOne, nanExceptionExponentIsInfinity);
2776             LValue absoluteBase = m_out.doubleAbs(base);
2777             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
2778             m_out.branch(absoluteBaseIsOne, rarely(nanExceptionExponentIsInfinity), usually(testExponentIsOneHalf));
2779
2780             //     Test if abs(y) == Infinity.
2781             m_out.appendTo(nanExceptionExponentIsInfinity, testExponentIsOneHalf);
2782             LValue absoluteExponent = m_out.doubleAbs(exponent);
2783             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
2784             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionResultIsNaN), usually(testExponentIsOneHalf));
2785
2786             // If y == 0.5 or y == -0.5, handle it through SQRT.
2787             // We have be carefuly with -0 and -Infinity.
2788
2789             //     Test if y == 0.5
2790             m_out.appendTo(testExponentIsOneHalf, handleBaseZeroExponentIsOneHalf);
2791             LValue exponentIsOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(0.5));
2792             m_out.branch(exponentIsOneHalf, rarely(handleBaseZeroExponentIsOneHalf), usually(testExponentIsNegativeOneHalf));
2793
2794             //     Handle x == -0.
2795             m_out.appendTo(handleBaseZeroExponentIsOneHalf, handleInfinityForExponentIsOneHalf);
2796             LValue baseIsZeroExponentIsOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2797             ValueFromBlock zeroResultExponentIsOneHalf = m_out.anchor(m_out.doubleZero);
2798             m_out.branch(baseIsZeroExponentIsOneHalf, rarely(continuation), usually(handleInfinityForExponentIsOneHalf));
2799
2800             //     Test if abs(x) == Infinity.
2801             m_out.appendTo(handleInfinityForExponentIsOneHalf, exponentIsOneHalfNormal);
2802             LValue absoluteBaseIsInfinityOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2803             m_out.branch(absoluteBaseIsInfinityOneHalf, rarely(exponentIsOneHalfInfinity), usually(exponentIsOneHalfNormal));
2804
2805             //     The exponent is 0.5, the base is finite or NaN, we can use SQRT.
2806             m_out.appendTo(exponentIsOneHalfNormal, exponentIsOneHalfInfinity);
2807             ValueFromBlock sqrtResult = m_out.anchor(m_out.doubleSqrt(base));
2808             m_out.jump(continuation);
2809
2810             //     The exponent is 0.5, the base is infinite, the result is always infinite.
2811             m_out.appendTo(exponentIsOneHalfInfinity, testExponentIsNegativeOneHalf);
2812             ValueFromBlock sqrtInfinityResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2813             m_out.jump(continuation);
2814
2815             //     Test if y == -0.5
2816             m_out.appendTo(testExponentIsNegativeOneHalf, testBaseZeroExponentIsNegativeOneHalf);
2817             LValue exponentIsNegativeOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(-0.5));
2818             m_out.branch(exponentIsNegativeOneHalf, rarely(testBaseZeroExponentIsNegativeOneHalf), usually(powBlock));
2819
2820             //     Handle x == -0.
2821             m_out.appendTo(testBaseZeroExponentIsNegativeOneHalf, handleBaseZeroExponentIsNegativeOneHalf);
2822             LValue baseIsZeroExponentIsNegativeOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2823             m_out.branch(baseIsZeroExponentIsNegativeOneHalf, rarely(handleBaseZeroExponentIsNegativeOneHalf), usually(handleInfinityForExponentIsNegativeOneHalf));
2824
2825             m_out.appendTo(handleBaseZeroExponentIsNegativeOneHalf, handleInfinityForExponentIsNegativeOneHalf);
2826             ValueFromBlock oneOverSqrtZeroResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2827             m_out.jump(continuation);
2828
2829             //     Test if abs(x) == Infinity.
2830             m_out.appendTo(handleInfinityForExponentIsNegativeOneHalf, exponentIsNegativeOneHalfNormal);
2831             LValue absoluteBaseIsInfinityNegativeOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2832             m_out.branch(absoluteBaseIsInfinityNegativeOneHalf, rarely(exponentIsNegativeOneHalfInfinity), usually(exponentIsNegativeOneHalfNormal));
2833
2834             //     The exponent is -0.5, the base is finite or NaN, we can use 1/SQRT.
2835             m_out.appendTo(exponentIsNegativeOneHalfNormal, exponentIsNegativeOneHalfInfinity);
2836             LValue sqrtBase = m_out.doubleSqrt(base);
2837             ValueFromBlock oneOverSqrtResult = m_out.anchor(m_out.div(m_out.constDouble(1.), sqrtBase));
2838             m_out.jump(continuation);
2839
2840             //     The exponent is -0.5, the base is infinite, the result is always zero.
2841             m_out.appendTo(exponentIsNegativeOneHalfInfinity, powBlock);
2842             ValueFromBlock oneOverSqrtInfinityResult = m_out.anchor(m_out.doubleZero);
2843             m_out.jump(continuation);
2844
2845             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
2846             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
2847             m_out.jump(continuation);
2848
2849             m_out.appendTo(nanExceptionResultIsNaN, continuation);
2850             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
2851             m_out.jump(continuation);
2852
2853             m_out.appendTo(continuation, lastNext);
2854             setDouble(m_out.phi(Double, powDoubleIntResult, zeroResultExponentIsOneHalf, sqrtResult, sqrtInfinityResult, oneOverSqrtZeroResult, oneOverSqrtResult, oneOverSqrtInfinityResult, powResult, pureNan));
2855         }
2856     }
2857
2858     void compileArithRandom()
2859     {
2860         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2861
2862         // Inlined WeakRandom::advance().
2863         // uint64_t x = m_low;
2864         void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
2865         LValue low = m_out.load64(m_out.absolute(lowAddress));
2866         // uint64_t y = m_high;
2867         void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();
2868         LValue high = m_out.load64(m_out.absolute(highAddress));
2869         // m_low = y;
2870         m_out.store64(high, m_out.absolute(lowAddress));
2871
2872         // x ^= x << 23;
2873         LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low);
2874
2875         // x ^= x >> 17;
2876         LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1);
2877
2878         // x ^= y ^ (y >> 26);
2879         LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2);
2880
2881         // m_high = x;
2882         m_out.store64(phase3, m_out.absolute(highAddress));
2883
2884         // return x + y;
2885         LValue random64 = m_out.add(phase3, high);
2886
2887         // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
2888         LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1));
2889
2890         LValue double53Integer = m_out.intToDouble(random53);
2891
2892         // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
2893         // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
2894         static const double scale = 1.0 / (1ULL << 53);
2895
2896         // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
2897         // It just reduces the exp part of the given 53bit double integer.
2898         // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
2899         // Now we get 53bit precision random double value in [0, 1).
2900         LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale));
2901
2902         setDouble(result);
2903     }
2904
2905     void compileArithRound()
2906     {
2907         if (m_node->child1().useKind() == DoubleRepUse) {
2908             LValue result = nullptr;
2909             if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) {
2910                 LValue value = lowDouble(m_node->child1());
2911                 result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5)));
2912             } else {
2913                 LBasicBlock realPartIsMoreThanHalf = m_out.newBlock();
2914                 LBasicBlock continuation = m_out.newBlock();
2915
2916                 LValue value = lowDouble(m_node->child1());
2917                 LValue integerValue = m_out.doubleCeil(value);
2918                 ValueFromBlock integerValueResult = m_out.anchor(integerValue);
2919
2920                 LValue realPart = m_out.doubleSub(integerValue, value);
2921
2922                 m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
2923
2924                 LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
2925                 LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
2926                 ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
2927                 m_out.jump(continuation);
2928                 m_out.appendTo(continuation, lastNext);
2929
2930                 result = m_out.phi(Double, integerValueResult, integerValueRoundedDownResult);
2931             }
2932
2933             if (producesInteger(m_node->arithRoundingMode())) {
2934                 LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
2935                 setInt32(integerValue);
2936             } else
2937                 setDouble(result);
2938             return;
2939         }
2940
2941         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2942         LValue argument = lowJSValue(m_node->child1());
2943         setJSValue(vmCall(Int64, m_out.operation(operationArithRound), m_callFrame, argument));
2944     }
2945
2946     void compileArithFloor()
2947     {
2948         if (m_node->child1().useKind() == DoubleRepUse) {
2949             LValue value = lowDouble(m_node->child1());
2950             LValue integerValue = m_out.doubleFloor(value);
2951             if (producesInteger(m_node->arithRoundingMode()))
2952                 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2953             else
2954                 setDouble(integerValue);
2955             return;
2956         }
2957         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2958         LValue argument = lowJSValue(m_node->child1());
2959         setJSValue(vmCall(Int64, m_out.operation(operationArithFloor), m_callFrame, argument));
2960     }
2961
2962     void compileArithCeil()
2963     {
2964         if (m_node->child1().useKind() == DoubleRepUse) {
2965             LValue value = lowDouble(m_node->child1());
2966             LValue integerValue = m_out.doubleCeil(value);
2967             if (producesInteger(m_node->arithRoundingMode()))
2968                 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2969             else
2970                 setDouble(integerValue);
2971             return;
2972         }
2973         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2974         LValue argument = lowJSValue(m_node->child1());
2975         setJSValue(vmCall(Int64, m_out.operation(operationArithCeil), m_callFrame, argument));
2976     }
2977
2978     void compileArithTrunc()
2979     {
2980         if (m_node->child1().useKind() == DoubleRepUse) {
2981             LValue value = lowDouble(m_node->child1());
2982             LValue result = m_out.doubleTrunc(value);
2983             if (producesInteger(m_node->arithRoundingMode()))
2984                 setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())));
2985             else
2986                 setDouble(result);
2987             return;
2988         }
2989         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2990         LValue argument = lowJSValue(m_node->child1());
2991         setJSValue(vmCall(Int64, m_out.operation(operationArithTrunc), m_callFrame, argument));
2992     }
2993
2994     void compileArithSqrt()
2995     {
2996         if (m_node->child1().useKind() == DoubleRepUse) {
2997             setDouble(m_out.doubleSqrt(lowDouble(m_node->child1())));
2998             return;
2999         }
3000         LValue argument = lowJSValue(m_node->child1());
3001         LValue result = vmCall(Double, m_out.operation(operationArithSqrt), m_callFrame, argument);
3002         setDouble(result);
3003     }
3004
3005     void compileArithFRound()
3006     {
3007         if (m_node->child1().useKind() == DoubleRepUse) {
3008             setDouble(m_out.fround(lowDouble(m_node->child1())));
3009             return;
3010         }
3011         LValue argument = lowJSValue(m_node->child1());
3012         LValue result = vmCall(Double, m_out.operation(operationArithFRound), m_callFrame, argument);
3013         setDouble(result);
3014     }
3015
3016     void compileValueNegate()
3017     {
3018         DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse);
3019         CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
3020         unsigned bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
3021         ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
3022         auto repatchingFunction = operationArithNegateOptimize;
3023         auto nonRepatchingFunction = operationArithNegate;
3024         compileUnaryMathIC<JITNegGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
3025     }
3026
3027     void compileArithNegate()
3028     {
3029         switch (m_node->child1().useKind()) {
3030         case Int32Use: {
3031             LValue value = lowInt32(m_node->child1());
3032             
3033             LValue result;
3034             if (!shouldCheckOverflow(m_node->arithMode()))
3035                 result = m_out.neg(value);
3036             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
3037                 CheckValue* check = m_out.speculateSub(m_out.int32Zero, value);
3038                 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
3039                 result = check;
3040             } else {
3041                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
3042                 result = m_out.neg(value);
3043             }
3044
3045             setInt32(result);
3046             break;
3047         }
3048             
3049         case Int52RepUse: {
3050             if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)) {
3051                 Int52Kind kind;
3052                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
3053                 LValue result = m_out.neg(value);
3054                 if (shouldCheckNegativeZero(m_node->arithMode()))
3055                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3056                 setInt52(result, kind);
3057                 break;
3058             }
3059             
3060             LValue value = lowInt52(m_node->child1());
3061             CheckValue* result = m_out.speculateSub(m_out.int64Zero, value);
3062             blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin);
3063             if (shouldCheckNegativeZero(m_node->arithMode()))
3064                 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3065             setInt52(result);
3066             break;
3067         }
3068             
3069         case DoubleRepUse: {
3070             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
3071             break;
3072         }
3073             
3074         default:
3075             DFG_CRASH(m_graph, m_node, "Bad use kind");
3076             break;
3077         }
3078     }
3079     
3080     void compileValueBitNot()
3081     {
3082         if (m_node->child1().useKind() == BigIntUse) {
3083             LValue operand = lowBigInt(m_node->child1());
3084             LValue result = vmCall(pointerType(), m_out.operation(operationBitNotBigInt), m_callFrame, operand);
3085             setJSValue(result);
3086             return;
3087         }
3088
3089         LValue operand = lowJSValue(m_node->child1());
3090         LValue result = vmCall(Int64, m_out.operation(operationValueBitNot), m_callFrame, operand);
3091         setJSValue(result);
3092     }
3093
3094     void compileArithBitNot()
3095     {
3096         setInt32(m_out.bitNot(lowInt32(m_node->child1())));
3097     }
3098
3099     void compileValueBitAnd()
3100     {
3101         if (m_node->isBinaryUseKind(BigIntUse)) {
3102             LValue left = lowBigInt(m_node->child1());
3103             LValue right = lowBigInt(m_node->child2());
3104             
3105             LValue result = vmCall(pointerType(), m_out.operation(operationBitAndBigInt), m_callFrame, left, right);
3106             setJSValue(result);
3107             return;
3108         }
3109         
3110         emitBinaryBitOpSnippet<JITBitAndGenerator>(operationValueBitAnd);
3111     }
3112     
3113     void compileArithBitAnd()
3114     {
3115         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3116     }
3117     
3118     void compileValueBitOr()
3119     {
3120         if (m_node->isBinaryUseKind(BigIntUse)) {
3121             LValue left = lowBigInt(m_node->child1());
3122             LValue right = lowBigInt(m_node->child2());
3123
3124             LValue result = vmCall(pointerType(), m_out.operation(operationBitOrBigInt), m_callFrame, left, right);
3125             setJSValue(result);
3126             return;
3127         }
3128
3129         emitBinaryBitOpSnippet<JITBitOrGenerator>(operationValueBitOr);
3130     }
3131
3132     void compileArithBitOr()
3133     {
3134         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3135     }
3136     
3137     void compileValueBitXor()
3138     {
3139         if (m_node->isBinaryUseKind(BigIntUse)) {
3140             LValue left = lowBigInt(m_node->child1());
3141             LValue right = lowBigInt(m_node->child2());
3142
3143             LValue result = vmCall(pointerType(), m_out.operation(operationBitXorBigInt), m_callFrame, left, right);
3144             setJSValue(result);
3145             return;
3146         }
3147
3148         emitBinaryBitOpSnippet<JITBitXorGenerator>(operationValueBitXor);
3149     }
3150
3151     void compileArithBitXor()
3152     {
3153         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3154     }
3155     
3156     void compileBitRShift()
3157     {
3158         if (m_node->isBinaryUseKind(UntypedUse)) {
3159             emitRightShiftSnippet(JITRightShiftGenerator::SignedShift);
3160             return;
3161         }
3162         setInt32(m_out.aShr(
3163             lowInt32(m_node->child1()),
3164             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3165     }
3166     
3167     void compileBitLShift()
3168     {
3169         if (m_node->isBinaryUseKind(UntypedUse)) {
3170             emitBinaryBitOpSnippet<JITLeftShiftGenerator>(operationValueBitLShift);
3171             return;
3172         }
3173         setInt32(m_out.shl(
3174             lowInt32(m_node->child1()),
3175             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3176     }
3177     
3178     void compileBitURShift()
3179     {
3180         if (m_node->isBinaryUseKind(UntypedUse)) {
3181             emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift);
3182             return;
3183         }
3184         setInt32(m_out.lShr(
3185             lowInt32(m_node->child1()),
3186             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3187     }
3188     
3189     void compileUInt32ToNumber()
3190     {
3191         LValue value = lowInt32(m_node->child1());
3192
3193         if (doesOverflow(m_node->arithMode())) {
3194             setStrictInt52(m_out.zeroExtPtr(value));
3195             return;
3196         }
3197
3198         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
3199         setInt32(value);
3200     }
3201     
3202     void compileCheckStructure()
3203     {
3204         ExitKind exitKind;
3205         if (m_node->child1()->hasConstant())
3206             exitKind = BadConstantCache;
3207         else
3208             exitKind = BadCache;
3209
3210         switch (m_node->child1().useKind()) {
3211         case CellUse:
3212         case KnownCellUse: {
3213             LValue cell = lowCell(m_node->child1());
3214             
3215             checkStructure(
3216                 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3217                 exitKind, m_node->structureSet(),
3218                 [&] (RegisteredStructure structure) {
3219                     return weakStructureID(structure);
3220                 });
3221             return;
3222         }
3223
3224         case CellOrOtherUse: {
3225             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
3226
3227             LBasicBlock cellCase = m_out.newBlock();
3228             LBasicBlock notCellCase = m_out.newBlock();
3229             LBasicBlock continuation = m_out.newBlock();
3230
3231             m_out.branch(
3232                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3233
3234             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3235             checkStructure(
3236                 m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value),
3237                 exitKind, m_node->structureSet(),
3238                 [&] (RegisteredStructure structure) {
3239                     return weakStructureID(structure);
3240                 });
3241             m_out.jump(continuation);
3242
3243             m_out.appendTo(notCellCase, continuation);
3244             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value));
3245             m_out.jump(continuation);
3246
3247             m_out.appendTo(continuation, lastNext);
3248             return;
3249         }
3250
3251         default:
3252             DFG_CRASH(m_graph, m_node, "Bad use kind");
3253             return;
3254         }
3255     }
3256
3257     void compileCheckStructureOrEmpty()
3258     {
3259         ExitKind exitKind;
3260         if (m_node->child1()->hasConstant())
3261             exitKind = BadConstantCache;
3262         else
3263             exitKind = BadCache;
3264
3265         LValue cell = lowCell(m_node->child1());
3266         bool maySeeEmptyValue = m_interpreter.forNode(m_node->child1()).m_type & SpecEmpty;
3267         LBasicBlock notEmpty;
3268         LBasicBlock continuation;
3269         LBasicBlock lastNext;
3270         if (maySeeEmptyValue) {
3271             notEmpty = m_out.newBlock();
3272             continuation = m_out.newBlock();
3273             m_out.branch(m_out.isZero64(cell), unsure(continuation), unsure(notEmpty));
3274             lastNext = m_out.appendTo(notEmpty, continuation);
3275         }
3276
3277         checkStructure(
3278             m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3279             exitKind, m_node->structureSet(),
3280             [&] (RegisteredStructure structure) {
3281                 return weakStructureID(structure);
3282             });
3283
3284         if (maySeeEmptyValue) {
3285             m_out.jump(continuation);
3286             m_out.appendTo(continuation, lastNext);
3287         }
3288     }
3289     
3290     void compileCheckCell()
3291     {
3292         LValue cell = lowCell(m_node->child1());
3293         
3294         speculate(
3295             BadCell, jsValueValue(cell), m_node->child1().node(),
3296             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
3297     }
3298     
3299     void compileCheckBadCell()
3300     {
3301         terminate(BadCell);
3302     }
3303
3304     void compileCheckNotEmpty()
3305     {
3306         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
3307     }
3308
3309     void compileAssertNotEmpty()
3310     {
3311         if (!validationEnabled())
3312             return;
3313
3314         LValue val = lowJSValue(m_node->child1());
3315         PatchpointValue* patchpoint = m_out.patchpoint(Void);
3316         patchpoint->appendSomeRegister(val);
3317         patchpoint->setGenerator(
3318             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
3319                 AllowMacroScratchRegisterUsage allowScratch(jit);
3320                 GPRReg input =  params[0].gpr();
3321                 CCallHelpers::Jump done = jit.branchIfNotEmpty(input);
3322                 jit.breakpoint();
3323                 done.link(&jit);
3324             });
3325     }
3326
3327     void compileCheckStringIdent()
3328     {
3329         UniquedStringImpl* uid = m_node->uidOperand();
3330         LValue stringImpl = lowStringIdent(m_node->child1());
3331         speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
3332     }
3333
3334     void compileGetExecutable()
3335     {
3336         LValue cell = lowCell(m_node->child1());
3337         speculateFunction(m_node->child1(), cell);
3338         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
3339     }
3340     
3341     void compileArrayify()
3342     {
3343         LValue cell = lowCell(m_node->child1());
3344         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
3345         
3346         LBasicBlock unexpectedStructure = m_out.newBlock();
3347         LBasicBlock continuation = m_out.newBlock();
3348         
3349         auto isUnexpectedArray = [&] (LValue cell) {
3350             if (m_node->op() == Arrayify)
3351                 return m_out.logicalNot(isArrayTypeForArrayify(cell, m_node->arrayMode()));
3352
3353             ASSERT(m_node->op() == ArrayifyToStructure);
3354             return m_out.notEqual(m_out.load32(cell, m_heaps.JSCell_structureID), weakStructureID(m_node->structure()));
3355         };
3356
3357         m_out.branch(isUnexpectedArray(cell), rarely(unexpectedStructure), usually(continuation));
3358
3359         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
3360         
3361         if (property) {
3362             switch (m_node->arrayMode().type()) {
3363             case Array::Int32:
3364             case Array::Double:
3365             case Array::Contiguous:
3366                 speculate(
3367                     Uncountable, noValue(), 0,
3368                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
3369                 break;
3370             default:
3371                 break;
3372             }
3373         }
3374         
3375         switch (m_node->arrayMode().type()) {
3376         case Array::Int32:
3377             vmCall(Void, m_out.operation(operationEnsureInt32), m_callFrame, cell);
3378             break;
3379         case Array::Double:
3380             vmCall(Void, m_out.operation(operationEnsureDouble), m_callFrame, cell);
3381             break;
3382         case Array::Contiguous:
3383             vmCall(Void, m_out.operation(operationEnsureContiguous), m_callFrame, cell);
3384             break;
3385         case Array::ArrayStorage:
3386         case Array::SlowPutArrayStorage:
3387             vmCall(Void, m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
3388             break;
3389         default:
3390             DFG_CRASH(m_graph, m_node, "Bad array type");
3391             break;
3392         }
3393         
3394         speculate(BadIndexingType, jsValueValue(cell), 0, isUnexpectedArray(cell));
3395         m_out.jump(continuation);
3396         
3397         m_out.appendTo(continuation, lastNext);
3398     }
3399     
3400     void compilePutStructure()
3401     {
3402         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
3403         
3404         RegisteredStructure oldStructure = m_node->transition()->previous;
3405         RegisteredStructure newStructure = m_node->transition()->next;
3406         ASSERT_UNUSED(oldStructure, oldStructure->indexingMode() == newStructure->indexingMode());
3407         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
3408         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
3409
3410         LValue cell = lowCell(m_node->child1()); 
3411         m_out.store32(
3412             weakStructureID(newStructure),
3413             cell, m_heaps.JSCell_structureID);
3414     }
3415     
3416     void compileGetById(AccessType type)
3417     {
3418         ASSERT(type == AccessType::Get || type == AccessType::TryGet || type == AccessType::GetDirect);
3419         switch (m_node->child1().useKind()) {
3420         case CellUse: {
3421             setJSValue(getById(lowCell(m_node->child1()), type));
3422             return;
3423         }
3424             
3425         case UntypedUse: {
3426             // This is pretty weird, since we duplicate the slow path both here and in the
3427             // code generated by the IC. We should investigate making this less bad.
3428             // https://bugs.webkit.org/show_bug.cgi?id=127830
3429             LValue value = lowJSValue(m_node->child1());
3430             
3431             LBasicBlock cellCase = m_out.newBlock();
3432             LBasicBlock notCellCase = m_out.newBlock();
3433             LBasicBlock continuation = m_out.newBlock();
3434             
3435             m_out.branch(
3436                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3437             
3438             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3439             ValueFromBlock cellResult = m_out.anchor(getById(value, type));
3440             m_out.jump(continuation);
3441
3442             J_JITOperation_EJI getByIdFunction = appropriateGenericGetByIdFunction(type);
3443
3444             m_out.appendTo(notCellCase, continuation);
3445             ValueFromBlock notCellResult = m_out.anchor(vmCall(
3446                 Int64, m_out.operation(getByIdFunction),
3447                 m_callFrame, value,
3448                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3449             m_out.jump(continuation);
3450             
3451             m_out.appendTo(continuation, lastNext);
3452             setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3453             return;
3454         }
3455             
3456         default:
3457             DFG_CRASH(m_graph, m_node, "Bad use kind");
3458             return;
3459         }
3460     }
3461
3462     void compileGetByIdWithThis()
3463     {
3464         if (m_node->child1().useKind() == CellUse && m_node->child2().useKind() == CellUse)
3465             setJSValue(getByIdWithThis(lowCell(m_node->child1()), lowCell(m_node->child2())));
3466         else {
3467             LValue base = lowJSValue(m_node->child1());
3468             LValue thisValue = lowJSValue(m_node->child2());
3469             
3470             LBasicBlock baseCellCase = m_out.newBlock();
3471             LBasicBlock notCellCase = m_out.newBlock();
3472             LBasicBlock thisValueCellCase = m_out.newBlock();
3473             LBasicBlock continuation = m_out.newBlock();
3474             
3475             m_out.branch(
3476                 isCell(base, provenType(m_node->child1())), unsure(baseCellCase), unsure(notCellCase));
3477             
3478             LBasicBlock lastNext = m_out.appendTo(baseCellCase, thisValueCellCase);
3479             
3480             m_out.branch(
3481                 isCell(thisValue, provenType(m_node->child2())), unsure(thisValueCellCase), unsure(notCellCase));
3482             
3483             m_out.appendTo(thisValueCellCase, notCellCase);
3484             ValueFromBlock cellResult = m_out.anchor(getByIdWithThis(base, thisValue));
3485             m_out.jump(continuation);
3486
3487             m_out.appendTo(notCellCase, continuation);
3488             ValueFromBlock notCellResult = m_out.anchor(vmCall(
3489                 Int64, m_out.operation(operationGetByIdWithThisGeneric),
3490                 m_callFrame, base, thisValue,
3491                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3492             m_out.jump(continuation);
3493             
3494             m_out.appendTo(continuation, lastNext);
3495             setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3496         }
3497         
3498     }
3499
3500     void compileGetByValWithThis()
3501     {
3502         LValue base = lowJSValue(m_node->child1());
3503         LValue thisValue = lowJSValue(m_node->child2());
3504         LValue subscript = lowJSValue(m_node->child3());
3505
3506         LValue result = vmCall(Int64, m_out.operation(operationGetByValWithThis), m_callFrame, base, thisValue, subscript);
3507         setJSValue(result);
3508     }
3509
3510     void compilePutByIdWithThis()
3511     {
3512         LValue base = lowJSValue(m_node->child1());
3513         LValue thisValue = lowJSValue(m_node->child2());
3514         LValue value = lowJSValue(m_node->child3());
3515
3516         vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis),
3517             m_callFrame, base, thisValue, value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()]));
3518     }
3519
3520     void compilePutByValWithThis()
3521     {
3522         LValue base = lowJSValue(m_graph.varArgChild(m_node, 0));
3523         LValue thisValue = lowJSValue(m_graph.varArgChild(m_node, 1));
3524         LValue property = lowJSValue(m_graph.varArgChild(m_node, 2));
3525         LValue value = lowJSValue(m_graph.varArgChild(m_node, 3));
3526
3527         vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis),
3528             m_callFrame, base, thisValue, property, value);
3529     }
3530     
3531     void compileAtomicsReadModifyWrite()
3532     {
3533         TypedArrayType type = m_node->arrayMode().typedArrayType();
3534         unsigned numExtraArgs = numExtraAtomicsArgs(m_node->op());
3535         Edge baseEdge = m_graph.child(m_node, 0);
3536         Edge indexEdge = m_graph.child(m_node, 1);
3537         Edge argEdges[maxNumExtraAtomicsArgs];
3538         for (unsigned i = numExtraArgs; i--;)
3539             argEdges[i] = m_graph.child(m_node, 2 + i);
3540         Edge storageEdge = m_graph.child(m_node, 2 + numExtraArgs);
3541         
3542         auto operation = [&] () -> LValue {
3543             switch (m_node->op()) {
3544             case AtomicsAdd:
3545                 return m_out.operation(operationAtomicsAdd);
3546             case AtomicsAnd:
3547                 return m_out.operation(operationAtomicsAnd);
3548             case AtomicsCompareExchange:
3549                 return m_out.operation(operationAtomicsCompareExchange);
3550             case AtomicsExchange:
3551                 return m_out.operation(operationAtomicsExchange);
3552             case AtomicsLoad:
3553                 return m_out.operation(operationAtomicsLoad);
3554             case AtomicsOr:
3555                 return m_out.operation(operationAtomicsOr);
3556             case AtomicsStore:
3557                 return m_out.operation(operationAtomicsStore);
3558             case AtomicsSub:
3559                 return m_out.operation(operationAtomicsSub);
3560             case AtomicsXor:
3561                 return m_out.operation(operationAtomicsXor);
3562             default:
3563                 RELEASE_ASSERT_NOT_REACHED();
3564                 break;
3565             }
3566         };
3567         
3568         if (!storageEdge) {
3569             Vector<LValue> args;
3570             args.append(m_callFrame);
3571             args.append(lowJSValue(baseEdge));
3572             args.append(lowJSValue(indexEdge));
3573             for (unsigned i = 0; i < numExtraArgs; ++i)
3574                 args.append(lowJSValue(argEdges[i]));
3575             LValue result = vmCall(Int64, operation(), args);
3576             setJSValue(result);
3577             return;
3578         }
3579
3580         LValue index = lowInt32(indexEdge);
3581         LValue args[2];
3582         for (unsigned i = numExtraArgs; i--;)
3583             args[i] = getIntTypedArrayStoreOperand(argEdges[i]);
3584         LValue storage = lowStorage(storageEdge);
3585
3586         TypedPointer pointer = pointerIntoTypedArray(storage, index, type);
3587         Width width = widthForBytes(elementSize(type));
3588         
3589         LValue atomicValue;
3590         LValue result;
3591         
3592         auto sanitizeResult = [&] (LValue value) -> LValue {
3593             if (isSigned(type)) {
3594                 switch (elementSize(type)) {
3595                 case 1:
3596                     value = m_out.bitAnd(value, m_out.constInt32(0xff));
3597                     break;
3598                 case 2:
3599                     value = m_out.bitAnd(value, m_out.constInt32(0xffff));
3600                     break;
3601                 case 4:
3602                     break;
3603                 default:
3604                     RELEASE_ASSERT_NOT_REACHED();
3605                     break;
3606                 }
3607             }
3608             return value;
3609         };
3610         
3611         switch (m_node->op()) {
3612         case AtomicsAdd:
3613             atomicValue = m_out.atomicXchgAdd(args[0], pointer, width);
3614             result = sanitizeResult(atomicValue);
3615             break;
3616         case AtomicsAnd:
3617             atomicValue = m_out.atomicXchgAnd(args[0], pointer, width);
3618             result = sanitizeResult(atomicValue);
3619             break;
3620         case AtomicsCompareExchange:
3621             atomicValue = m_out.atomicStrongCAS(args[0], args[1], pointer, width);
3622             result = sanitizeResult(atomicValue);
3623             break;
3624         case AtomicsExchange:
3625             atomicValue = m_out.atomicXchg(args[0], pointer, width);
3626             result = sanitizeResult(atomicValue);
3627             break;
3628         case AtomicsLoad:
3629             atomicValue = m_out.atomicXchgAdd(m_out.int32Zero,&n