Source/JavaScriptCore:
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLLowerDFGToB3.cpp
1 /*
2  * Copyright (C) 2013-2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLLowerDFGToB3.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "AirGenerationContext.h"
32 #include "AllowMacroScratchRegisterUsage.h"
33 #include "B3StackmapGenerationParams.h"
34 #include "CallFrameShuffler.h"
35 #include "CodeBlockWithJITType.h"
36 #include "DFGAbstractInterpreterInlines.h"
37 #include "DFGDominators.h"
38 #include "DFGInPlaceAbstractState.h"
39 #include "DFGOSRAvailabilityAnalysisPhase.h"
40 #include "DFGOSRExitFuzz.h"
41 #include "DirectArguments.h"
42 #include "FTLAbstractHeapRepository.h"
43 #include "FTLAvailableRecovery.h"
44 #include "FTLExceptionTarget.h"
45 #include "FTLForOSREntryJITCode.h"
46 #include "FTLFormattedValue.h"
47 #include "FTLLazySlowPathCall.h"
48 #include "FTLLoweredNodeValue.h"
49 #include "FTLOperations.h"
50 #include "FTLOutput.h"
51 #include "FTLPatchpointExceptionHandle.h"
52 #include "FTLThunks.h"
53 #include "FTLWeightedTarget.h"
54 #include "JITAddGenerator.h"
55 #include "JITBitAndGenerator.h"
56 #include "JITBitOrGenerator.h"
57 #include "JITBitXorGenerator.h"
58 #include "JITDivGenerator.h"
59 #include "JITInlineCacheGenerator.h"
60 #include "JITLeftShiftGenerator.h"
61 #include "JITMulGenerator.h"
62 #include "JITRightShiftGenerator.h"
63 #include "JITSubGenerator.h"
64 #include "JSCInlines.h"
65 #include "JSGeneratorFunction.h"
66 #include "JSLexicalEnvironment.h"
67 #include "OperandsInlines.h"
68 #include "ScopedArguments.h"
69 #include "ScopedArgumentsTable.h"
70 #include "ScratchRegisterAllocator.h"
71 #include "SetupVarargsFrame.h"
72 #include "VirtualRegister.h"
73 #include "Watchdog.h"
74 #include <atomic>
75 #if !OS(WINDOWS)
76 #include <dlfcn.h>
77 #endif
78 #include <unordered_set>
79 #include <wtf/Box.h>
80 #include <wtf/ProcessID.h>
81
82 namespace JSC { namespace FTL {
83
84 using namespace B3;
85 using namespace DFG;
86
87 namespace {
88
89 std::atomic<int> compileCounter;
90
91 #if ASSERT_DISABLED
92 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
93 {
94     CRASH();
95 }
96 #else
97 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
98     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
99 {
100     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
101     if (nodeIndex != UINT_MAX)
102         dataLog(", node @", nodeIndex);
103     dataLog(".\n");
104     CRASH();
105 }
106 #endif
107
108 // Using this instead of typeCheck() helps to reduce the load on B3, by creating
109 // significantly less dead code.
110 #define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \
111         FormattedValue _ftc_lowValue = (lowValue);                      \
112         Edge _ftc_highValue = (highValue);                              \
113         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
114         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
115             break;                                                      \
116         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \
117     } while (false)
118
119 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \
120     FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition)
121
122 class LowerDFGToB3 {
123     WTF_MAKE_NONCOPYABLE(LowerDFGToB3);
124 public:
125     LowerDFGToB3(State& state)
126         : m_graph(state.graph)
127         , m_ftlState(state)
128         , m_out(state)
129         , m_proc(*state.proc)
130         , m_state(state.graph)
131         , m_interpreter(state.graph, m_state)
132     {
133     }
134     
135     void lower()
136     {
137         State* state = &m_ftlState;
138         
139         CString name;
140         if (verboseCompilationEnabled()) {
141             name = toCString(
142                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
143                 "_", codeBlock()->hash());
144         } else
145             name = "jsBody";
146         
147         m_graph.ensureDominators();
148
149         if (verboseCompilationEnabled())
150             dataLog("Function ready, beginning lowering.\n");
151
152         m_out.initialize(m_heaps);
153
154         // We use prologue frequency for all of the initialization code.
155         m_out.setFrequency(1);
156         
157         m_prologue = m_out.newBlock();
158         LBasicBlock stackOverflow = m_out.newBlock();
159         m_handleExceptions = m_out.newBlock();
160         
161         LBasicBlock checkArguments = m_out.newBlock();
162
163         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
164             m_highBlock = m_graph.block(blockIndex);
165             if (!m_highBlock)
166                 continue;
167             m_out.setFrequency(m_highBlock->executionCount);
168             m_blocks.add(m_highBlock, m_out.newBlock());
169         }
170
171         // Back to prologue frequency for any bocks that get sneakily created in the initialization code.
172         m_out.setFrequency(1);
173         
174         m_out.appendTo(m_prologue, stackOverflow);
175         m_out.initializeConstants(m_proc, m_prologue);
176         createPhiVariables();
177
178         size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;
179         B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);
180         m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));
181         state->capturedValue = capturedBase->slot();
182
183         auto preOrder = m_graph.blocksInPreOrder();
184
185         // We should not create any alloca's after this point, since they will cease to
186         // be mem2reg candidates.
187         
188         m_callFrame = m_out.framePointer();
189         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
190         m_tagMask = m_out.constInt64(TagMask);
191
192         // Make sure that B3 knows that we really care about the mask registers. This forces the
193         // constants to be materialized in registers.
194         m_proc.addFastConstant(m_tagTypeNumber->key());
195         m_proc.addFastConstant(m_tagMask->key());
196         
197         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
198         
199         m_out.branch(
200             didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
201         
202         m_out.appendTo(stackOverflow, m_handleExceptions);
203         m_out.call(m_out.voidType, m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
204         m_out.patchpoint(Void)->setGenerator(
205             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
206                 // We are terminal, so we can clobber everything. That's why we don't claim to
207                 // clobber scratch.
208                 AllowMacroScratchRegisterUsage allowScratch(jit);
209                 
210                 jit.copyCalleeSavesToVMCalleeSavesBuffer();
211                 jit.move(CCallHelpers::TrustedImmPtr(jit.vm()), GPRInfo::argumentGPR0);
212                 jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
213                 CCallHelpers::Call call = jit.call();
214                 jit.jumpToExceptionHandler();
215
216                 jit.addLinkTask(
217                     [=] (LinkBuffer& linkBuffer) {
218                         linkBuffer.link(call, FunctionPtr(lookupExceptionHandlerFromCallerFrame));
219                     });
220             });
221         m_out.unreachable();
222         
223         m_out.appendTo(m_handleExceptions, checkArguments);
224         Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;
225         m_out.patchpoint(Void)->setGenerator(
226             [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
227                 CCallHelpers::Jump jump = jit.jump();
228                 jit.addLinkTask(
229                     [=] (LinkBuffer& linkBuffer) {
230                         linkBuffer.link(jump, linkBuffer.locationOf(*exceptionHandler));
231                     });
232             });
233         m_out.unreachable();
234         
235         m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
236         availabilityMap().clear();
237         availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
238         for (unsigned i = codeBlock()->numParameters(); i--;) {
239             availabilityMap().m_locals.argument(i) =
240                 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
241         }
242         m_node = nullptr;
243         m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
244         for (unsigned i = codeBlock()->numParameters(); i--;) {
245             Node* node = m_graph.m_arguments[i];
246             VirtualRegister operand = virtualRegisterForArgument(i);
247             
248             LValue jsValue = m_out.load64(addressFor(operand));
249             
250             if (node) {
251                 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
252                 
253                 // This is a hack, but it's an effective one. It allows us to do CSE on the
254                 // primordial load of arguments. This assumes that the GetLocal that got put in
255                 // place of the original SetArgument doesn't have any effects before it. This
256                 // should hold true.
257                 m_loadedArgumentValues.add(node, jsValue);
258             }
259             
260             switch (m_graph.m_argumentFormats[i]) {
261             case FlushedInt32:
262                 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
263                 break;
264             case FlushedBoolean:
265                 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
266                 break;
267             case FlushedCell:
268                 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
269                 break;
270             case FlushedJSValue:
271                 break;
272             default:
273                 DFG_CRASH(m_graph, node, "Bad flush format for argument");
274                 break;
275             }
276         }
277         m_out.jump(lowBlock(m_graph.block(0)));
278         
279         for (DFG::BasicBlock* block : preOrder)
280             compileBlock(block);
281
282         // Make sure everything is decorated. This does a bunch of deferred decorating. This has
283         // to happen last because our abstract heaps are generated lazily. They have to be
284         // generated lazily because we have an infiniten number of numbered, indexed, and
285         // absolute heaps. We only become aware of the ones we actually mention while lowering.
286         m_heaps.computeRangesAndDecorateInstructions();
287
288         // We create all Phi's up front, but we may then decide not to compile the basic block
289         // that would have contained one of them. So this creates orphans, which triggers B3
290         // validation failures. Calling this fixes the issue.
291         //
292         // Note that you should avoid the temptation to make this call conditional upon
293         // validation being enabled. B3 makes no guarantees of any kind of correctness when
294         // dealing with IR that would have failed validation. For example, it would be valid to
295         // write a B3 phase that so aggressively assumes the lack of orphans that it would crash
296         // if any orphans were around. We might even have such phases already.
297         m_proc.deleteOrphans();
298
299         // We put the blocks into the B3 procedure in a super weird order. Now we reorder them.
300         m_out.applyBlockOrder();
301     }
302
303 private:
304     
305     void createPhiVariables()
306     {
307         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
308             DFG::BasicBlock* block = m_graph.block(blockIndex);
309             if (!block)
310                 continue;
311             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
312                 Node* node = block->at(nodeIndex);
313                 if (node->op() != DFG::Phi)
314                     continue;
315                 LType type;
316                 switch (node->flags() & NodeResultMask) {
317                 case NodeResultDouble:
318                     type = m_out.doubleType;
319                     break;
320                 case NodeResultInt32:
321                     type = m_out.int32;
322                     break;
323                 case NodeResultInt52:
324                     type = m_out.int64;
325                     break;
326                 case NodeResultBoolean:
327                     type = m_out.boolean;
328                     break;
329                 case NodeResultJS:
330                     type = m_out.int64;
331                     break;
332                 default:
333                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
334                     break;
335                 }
336                 m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));
337             }
338         }
339     }
340     
341     void compileBlock(DFG::BasicBlock* block)
342     {
343         if (!block)
344             return;
345         
346         if (verboseCompilationEnabled())
347             dataLog("Compiling block ", *block, "\n");
348         
349         m_highBlock = block;
350         
351         // Make sure that any blocks created while lowering code in the high block have the frequency of
352         // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs
353         // something roughly approximate for things like register allocation.
354         m_out.setFrequency(m_highBlock->executionCount);
355         
356         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
357         
358         m_nextHighBlock = 0;
359         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
360             m_nextHighBlock = m_graph.block(nextBlockIndex);
361             if (m_nextHighBlock)
362                 break;
363         }
364         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
365         
366         // All of this effort to find the next block gives us the ability to keep the
367         // generated IR in roughly program order. This ought not affect the performance
368         // of the generated code (since we expect B3 to reorder things) but it will
369         // make IR dumps easier to read.
370         m_out.appendTo(lowBlock, m_nextLowBlock);
371         
372         if (Options::ftlCrashes())
373             m_out.trap();
374         
375         if (!m_highBlock->cfaHasVisited) {
376             if (verboseCompilationEnabled())
377                 dataLog("Bailing because CFA didn't reach.\n");
378             crash(m_highBlock->index, UINT_MAX);
379             return;
380         }
381         
382         m_availabilityCalculator.beginBlock(m_highBlock);
383         
384         m_state.reset();
385         m_state.beginBasicBlock(m_highBlock);
386         
387         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
388             if (!compileNode(m_nodeIndex))
389                 break;
390         }
391     }
392
393     void safelyInvalidateAfterTermination()
394     {
395         if (verboseCompilationEnabled())
396             dataLog("Bailing.\n");
397         crash();
398
399         // Invalidate dominated blocks. Under normal circumstances we would expect
400         // them to be invalidated already. But you can have the CFA become more
401         // precise over time because the structures of objects change on the main
402         // thread. Failing to do this would result in weird crashes due to a value
403         // being used but not defined. Race conditions FTW!
404         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
405             DFG::BasicBlock* target = m_graph.block(blockIndex);
406             if (!target)
407                 continue;
408             if (m_graph.m_dominators->dominates(m_highBlock, target)) {
409                 if (verboseCompilationEnabled())
410                     dataLog("Block ", *target, " will bail also.\n");
411                 target->cfaHasVisited = false;
412             }
413         }
414     }
415
416     bool compileNode(unsigned nodeIndex)
417     {
418         if (!m_state.isValid()) {
419             safelyInvalidateAfterTermination();
420             return false;
421         }
422         
423         m_node = m_highBlock->at(nodeIndex);
424         m_origin = m_node->origin;
425         m_out.setOrigin(m_node);
426         
427         if (verboseCompilationEnabled())
428             dataLog("Lowering ", m_node, "\n");
429         
430         m_availableRecoveries.resize(0);
431         
432         m_interpreter.startExecuting();
433         
434         switch (m_node->op()) {
435         case DFG::Upsilon:
436             compileUpsilon();
437             break;
438         case DFG::Phi:
439             compilePhi();
440             break;
441         case JSConstant:
442             break;
443         case DoubleConstant:
444             compileDoubleConstant();
445             break;
446         case Int52Constant:
447             compileInt52Constant();
448             break;
449         case LazyJSConstant:
450             compileLazyJSConstant();
451             break;
452         case DoubleRep:
453             compileDoubleRep();
454             break;
455         case DoubleAsInt32:
456             compileDoubleAsInt32();
457             break;
458         case DFG::ValueRep:
459             compileValueRep();
460             break;
461         case Int52Rep:
462             compileInt52Rep();
463             break;
464         case ValueToInt32:
465             compileValueToInt32();
466             break;
467         case BooleanToNumber:
468             compileBooleanToNumber();
469             break;
470         case ExtractOSREntryLocal:
471             compileExtractOSREntryLocal();
472             break;
473         case GetStack:
474             compileGetStack();
475             break;
476         case PutStack:
477             compilePutStack();
478             break;
479         case DFG::Check:
480             compileNoOp();
481             break;
482         case ToThis:
483             compileToThis();
484             break;
485         case ValueAdd:
486             compileValueAdd();
487             break;
488         case StrCat:
489             compileStrCat();
490             break;
491         case ArithAdd:
492         case ArithSub:
493             compileArithAddOrSub();
494             break;
495         case ArithClz32:
496             compileArithClz32();
497             break;
498         case ArithMul:
499             compileArithMul();
500             break;
501         case ArithDiv:
502             compileArithDiv();
503             break;
504         case ArithMod:
505             compileArithMod();
506             break;
507         case ArithMin:
508         case ArithMax:
509             compileArithMinOrMax();
510             break;
511         case ArithAbs:
512             compileArithAbs();
513             break;
514         case ArithSin:
515             compileArithSin();
516             break;
517         case ArithCos:
518             compileArithCos();
519             break;
520         case ArithPow:
521             compileArithPow();
522             break;
523         case ArithRandom:
524             compileArithRandom();
525             break;
526         case ArithRound:
527             compileArithRound();
528             break;
529         case ArithFloor:
530             compileArithFloor();
531             break;
532         case ArithCeil:
533             compileArithCeil();
534             break;
535         case ArithTrunc:
536             compileArithTrunc();
537             break;
538         case ArithSqrt:
539             compileArithSqrt();
540             break;
541         case ArithLog:
542             compileArithLog();
543             break;
544         case ArithFRound:
545             compileArithFRound();
546             break;
547         case ArithNegate:
548             compileArithNegate();
549             break;
550         case DFG::BitAnd:
551             compileBitAnd();
552             break;
553         case DFG::BitOr:
554             compileBitOr();
555             break;
556         case DFG::BitXor:
557             compileBitXor();
558             break;
559         case BitRShift:
560             compileBitRShift();
561             break;
562         case BitLShift:
563             compileBitLShift();
564             break;
565         case BitURShift:
566             compileBitURShift();
567             break;
568         case UInt32ToNumber:
569             compileUInt32ToNumber();
570             break;
571         case CheckStructure:
572             compileCheckStructure();
573             break;
574         case CheckCell:
575             compileCheckCell();
576             break;
577         case CheckNotEmpty:
578             compileCheckNotEmpty();
579             break;
580         case CheckBadCell:
581             compileCheckBadCell();
582             break;
583         case CheckIdent:
584             compileCheckIdent();
585             break;
586         case GetExecutable:
587             compileGetExecutable();
588             break;
589         case ArrayifyToStructure:
590             compileArrayifyToStructure();
591             break;
592         case PutStructure:
593             compilePutStructure();
594             break;
595         case GetById:
596         case GetByIdFlush:
597             compileGetById();
598             break;
599         case In:
600             compileIn();
601             break;
602         case PutById:
603         case PutByIdDirect:
604         case PutByIdFlush:
605             compilePutById();
606             break;
607         case PutGetterById:
608         case PutSetterById:
609             compilePutAccessorById();
610             break;
611         case PutGetterSetterById:
612             compilePutGetterSetterById();
613             break;
614         case PutGetterByVal:
615         case PutSetterByVal:
616             compilePutAccessorByVal();
617             break;
618         case GetButterfly:
619             compileGetButterfly();
620             break;
621         case ConstantStoragePointer:
622             compileConstantStoragePointer();
623             break;
624         case GetIndexedPropertyStorage:
625             compileGetIndexedPropertyStorage();
626             break;
627         case CheckArray:
628             compileCheckArray();
629             break;
630         case GetArrayLength:
631             compileGetArrayLength();
632             break;
633         case CheckInBounds:
634             compileCheckInBounds();
635             break;
636         case GetByVal:
637             compileGetByVal();
638             break;
639         case GetMyArgumentByVal:
640             compileGetMyArgumentByVal();
641             break;
642         case PutByVal:
643         case PutByValAlias:
644         case PutByValDirect:
645             compilePutByVal();
646             break;
647         case ArrayPush:
648             compileArrayPush();
649             break;
650         case ArrayPop:
651             compileArrayPop();
652             break;
653         case CreateActivation:
654             compileCreateActivation();
655             break;
656         case NewFunction:
657         case NewArrowFunction:
658         case NewGeneratorFunction:
659             compileNewFunction();
660             break;
661         case CreateDirectArguments:
662             compileCreateDirectArguments();
663             break;
664         case CreateScopedArguments:
665             compileCreateScopedArguments();
666             break;
667         case CreateClonedArguments:
668             compileCreateClonedArguments();
669             break;
670         case NewObject:
671             compileNewObject();
672             break;
673         case NewArray:
674             compileNewArray();
675             break;
676         case NewArrayBuffer:
677             compileNewArrayBuffer();
678             break;
679         case NewArrayWithSize:
680             compileNewArrayWithSize();
681             break;
682         case NewTypedArray:
683             compileNewTypedArray();
684             break;
685         case GetTypedArrayByteOffset:
686             compileGetTypedArrayByteOffset();
687             break;
688         case AllocatePropertyStorage:
689             compileAllocatePropertyStorage();
690             break;
691         case ReallocatePropertyStorage:
692             compileReallocatePropertyStorage();
693             break;
694         case ToString:
695         case CallStringConstructor:
696             compileToStringOrCallStringConstructor();
697             break;
698         case ToPrimitive:
699             compileToPrimitive();
700             break;
701         case MakeRope:
702             compileMakeRope();
703             break;
704         case StringCharAt:
705             compileStringCharAt();
706             break;
707         case StringCharCodeAt:
708             compileStringCharCodeAt();
709             break;
710         case StringFromCharCode:
711             compileStringFromCharCode();
712             break;
713         case GetByOffset:
714         case GetGetterSetterByOffset:
715             compileGetByOffset();
716             break;
717         case GetGetter:
718             compileGetGetter();
719             break;
720         case GetSetter:
721             compileGetSetter();
722             break;
723         case MultiGetByOffset:
724             compileMultiGetByOffset();
725             break;
726         case PutByOffset:
727             compilePutByOffset();
728             break;
729         case MultiPutByOffset:
730             compileMultiPutByOffset();
731             break;
732         case GetGlobalVar:
733         case GetGlobalLexicalVariable:
734             compileGetGlobalVariable();
735             break;
736         case PutGlobalVariable:
737             compilePutGlobalVariable();
738             break;
739         case NotifyWrite:
740             compileNotifyWrite();
741             break;
742         case GetCallee:
743             compileGetCallee();
744             break;
745         case GetArgumentCount:
746             compileGetArgumentCount();
747             break;
748         case GetScope:
749             compileGetScope();
750             break;
751         case SkipScope:
752             compileSkipScope();
753             break;
754         case GetGlobalObject:
755             compileGetGlobalObject();
756             break;
757         case GetClosureVar:
758             compileGetClosureVar();
759             break;
760         case PutClosureVar:
761             compilePutClosureVar();
762             break;
763         case GetFromArguments:
764             compileGetFromArguments();
765             break;
766         case PutToArguments:
767             compilePutToArguments();
768             break;
769         case CompareEq:
770             compileCompareEq();
771             break;
772         case CompareStrictEq:
773             compileCompareStrictEq();
774             break;
775         case CompareLess:
776             compileCompareLess();
777             break;
778         case CompareLessEq:
779             compileCompareLessEq();
780             break;
781         case CompareGreater:
782             compileCompareGreater();
783             break;
784         case CompareGreaterEq:
785             compileCompareGreaterEq();
786             break;
787         case LogicalNot:
788             compileLogicalNot();
789             break;
790         case Call:
791         case TailCallInlinedCaller:
792         case Construct:
793             compileCallOrConstruct();
794             break;
795         case TailCall:
796             compileTailCall();
797             break;
798         case CallVarargs:
799         case CallForwardVarargs:
800         case TailCallVarargs:
801         case TailCallVarargsInlinedCaller:
802         case TailCallForwardVarargs:
803         case TailCallForwardVarargsInlinedCaller:
804         case ConstructVarargs:
805         case ConstructForwardVarargs:
806             compileCallOrConstructVarargs();
807             break;
808         case LoadVarargs:
809             compileLoadVarargs();
810             break;
811         case ForwardVarargs:
812             compileForwardVarargs();
813             break;
814         case DFG::Jump:
815             compileJump();
816             break;
817         case DFG::Branch:
818             compileBranch();
819             break;
820         case DFG::Switch:
821             compileSwitch();
822             break;
823         case DFG::Return:
824             compileReturn();
825             break;
826         case ForceOSRExit:
827             compileForceOSRExit();
828             break;
829         case Throw:
830         case ThrowReferenceError:
831             compileThrow();
832             break;
833         case InvalidationPoint:
834             compileInvalidationPoint();
835             break;
836         case IsUndefined:
837             compileIsUndefined();
838             break;
839         case IsBoolean:
840             compileIsBoolean();
841             break;
842         case IsNumber:
843             compileIsNumber();
844             break;
845         case IsString:
846             compileIsString();
847             break;
848         case IsObject:
849             compileIsObject();
850             break;
851         case IsObjectOrNull:
852             compileIsObjectOrNull();
853             break;
854         case IsFunction:
855             compileIsFunction();
856             break;
857         case TypeOf:
858             compileTypeOf();
859             break;
860         case CheckTypeInfoFlags:
861             compileCheckTypeInfoFlags();
862             break;
863         case OverridesHasInstance:
864             compileOverridesHasInstance();
865             break;
866         case InstanceOf:
867             compileInstanceOf();
868             break;
869         case InstanceOfCustom:
870             compileInstanceOfCustom();
871             break;
872         case CountExecution:
873             compileCountExecution();
874             break;
875         case StoreBarrier:
876             compileStoreBarrier();
877             break;
878         case HasIndexedProperty:
879             compileHasIndexedProperty();
880             break;
881         case HasGenericProperty:
882             compileHasGenericProperty();
883             break;
884         case HasStructureProperty:
885             compileHasStructureProperty();
886             break;
887         case GetDirectPname:
888             compileGetDirectPname();
889             break;
890         case GetEnumerableLength:
891             compileGetEnumerableLength();
892             break;
893         case GetPropertyEnumerator:
894             compileGetPropertyEnumerator();
895             break;
896         case GetEnumeratorStructurePname:
897             compileGetEnumeratorStructurePname();
898             break;
899         case GetEnumeratorGenericPname:
900             compileGetEnumeratorGenericPname();
901             break;
902         case ToIndexString:
903             compileToIndexString();
904             break;
905         case CheckStructureImmediate:
906             compileCheckStructureImmediate();
907             break;
908         case MaterializeNewObject:
909             compileMaterializeNewObject();
910             break;
911         case MaterializeCreateActivation:
912             compileMaterializeCreateActivation();
913             break;
914         case CheckWatchdogTimer:
915             compileCheckWatchdogTimer();
916             break;
917         case CopyRest:
918             compileCopyRest();
919             break;
920         case GetRestLength:
921             compileGetRestLength();
922             break;
923         case RegExpExec:
924             compileRegExpExec();
925             break;
926         case RegExpTest:
927             compileRegExpTest();
928             break;
929         case NewRegexp:
930             compileNewRegexp();
931             break;
932         case SetFunctionName:
933             compileSetFunctionName();
934             break;
935         case StringReplace:
936             compileStringReplace();
937             break;
938         case GetRegExpObjectLastIndex:
939             compileGetRegExpObjectLastIndex();
940             break;
941         case SetRegExpObjectLastIndex:
942             compileSetRegExpObjectLastIndex();
943             break;
944         case RecordRegExpCachedResult:
945             compileRecordRegExpCachedResult();
946             break;
947
948         case PhantomLocal:
949         case LoopHint:
950         case MovHint:
951         case ZombieHint:
952         case ExitOK:
953         case PhantomNewObject:
954         case PhantomNewFunction:
955         case PhantomNewGeneratorFunction:
956         case PhantomCreateActivation:
957         case PhantomDirectArguments:
958         case PhantomClonedArguments:
959         case PutHint:
960         case BottomValue:
961         case KillStack:
962             break;
963         default:
964             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
965             break;
966         }
967         
968         if (m_node->isTerminal())
969             return false;
970         
971         if (!m_state.isValid()) {
972             safelyInvalidateAfterTermination();
973             return false;
974         }
975
976         m_availabilityCalculator.executeNode(m_node);
977         m_interpreter.executeEffects(nodeIndex);
978         
979         return true;
980     }
981
982     void compileUpsilon()
983     {
984         LValue upsilonValue = nullptr;
985         switch (m_node->child1().useKind()) {
986         case DoubleRepUse:
987             upsilonValue = lowDouble(m_node->child1());
988             break;
989         case Int32Use:
990         case KnownInt32Use:
991             upsilonValue = lowInt32(m_node->child1());
992             break;
993         case Int52RepUse:
994             upsilonValue = lowInt52(m_node->child1());
995             break;
996         case BooleanUse:
997         case KnownBooleanUse:
998             upsilonValue = lowBoolean(m_node->child1());
999             break;
1000         case CellUse:
1001         case KnownCellUse:
1002             upsilonValue = lowCell(m_node->child1());
1003             break;
1004         case UntypedUse:
1005             upsilonValue = lowJSValue(m_node->child1());
1006             break;
1007         default:
1008             DFG_CRASH(m_graph, m_node, "Bad use kind");
1009             break;
1010         }
1011         ValueFromBlock upsilon = m_out.anchor(upsilonValue);
1012         LValue phiNode = m_phis.get(m_node->phi());
1013         m_out.addIncomingToPhi(phiNode, upsilon);
1014     }
1015     
1016     void compilePhi()
1017     {
1018         LValue phi = m_phis.get(m_node);
1019         m_out.m_block->append(phi);
1020
1021         switch (m_node->flags() & NodeResultMask) {
1022         case NodeResultDouble:
1023             setDouble(phi);
1024             break;
1025         case NodeResultInt32:
1026             setInt32(phi);
1027             break;
1028         case NodeResultInt52:
1029             setInt52(phi);
1030             break;
1031         case NodeResultBoolean:
1032             setBoolean(phi);
1033             break;
1034         case NodeResultJS:
1035             setJSValue(phi);
1036             break;
1037         default:
1038             DFG_CRASH(m_graph, m_node, "Bad use kind");
1039             break;
1040         }
1041     }
1042     
1043     void compileDoubleConstant()
1044     {
1045         setDouble(m_out.constDouble(m_node->asNumber()));
1046     }
1047     
1048     void compileInt52Constant()
1049     {
1050         int64_t value = m_node->asMachineInt();
1051         
1052         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
1053         setStrictInt52(m_out.constInt64(value));
1054     }
1055
1056     void compileLazyJSConstant()
1057     {
1058         PatchpointValue* patchpoint = m_out.patchpoint(Int64);
1059         LazyJSValue value = m_node->lazyJSValue();
1060         patchpoint->setGenerator(
1061             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1062                 value.emit(jit, JSValueRegs(params[0].gpr()));
1063             });
1064         patchpoint->effects = Effects::none();
1065         setJSValue(patchpoint);
1066     }
1067
1068     void compileDoubleRep()
1069     {
1070         switch (m_node->child1().useKind()) {
1071         case RealNumberUse: {
1072             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1073             
1074             LValue doubleValue = unboxDouble(value);
1075             
1076             LBasicBlock intCase = m_out.newBlock();
1077             LBasicBlock continuation = m_out.newBlock();
1078             
1079             ValueFromBlock fastResult = m_out.anchor(doubleValue);
1080             m_out.branch(
1081                 m_out.doubleEqual(doubleValue, doubleValue),
1082                 usually(continuation), rarely(intCase));
1083             
1084             LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
1085             
1086             FTL_TYPE_CHECK(
1087                 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
1088                 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
1089             ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
1090             m_out.jump(continuation);
1091             
1092             m_out.appendTo(continuation, lastNext);
1093             
1094             setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult));
1095             return;
1096         }
1097             
1098         case NotCellUse:
1099         case NumberUse: {
1100             bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
1101             
1102             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1103
1104             LBasicBlock intCase = m_out.newBlock();
1105             LBasicBlock doubleTesting = m_out.newBlock();
1106             LBasicBlock doubleCase = m_out.newBlock();
1107             LBasicBlock nonDoubleCase = m_out.newBlock();
1108             LBasicBlock continuation = m_out.newBlock();
1109             
1110             m_out.branch(
1111                 isNotInt32(value, provenType(m_node->child1())),
1112                 unsure(doubleTesting), unsure(intCase));
1113             
1114             LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1115             
1116             ValueFromBlock intToDouble = m_out.anchor(
1117                 m_out.intToDouble(unboxInt32(value)));
1118             m_out.jump(continuation);
1119             
1120             m_out.appendTo(doubleTesting, doubleCase);
1121             LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1122             m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1123
1124             m_out.appendTo(doubleCase, nonDoubleCase);
1125             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1126             m_out.jump(continuation);
1127
1128             if (shouldConvertNonNumber) {
1129                 LBasicBlock undefinedCase = m_out.newBlock();
1130                 LBasicBlock testNullCase = m_out.newBlock();
1131                 LBasicBlock nullCase = m_out.newBlock();
1132                 LBasicBlock testBooleanTrueCase = m_out.newBlock();
1133                 LBasicBlock convertBooleanTrueCase = m_out.newBlock();
1134                 LBasicBlock convertBooleanFalseCase = m_out.newBlock();
1135
1136                 m_out.appendTo(nonDoubleCase, undefinedCase);
1137                 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1138                 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1139
1140                 m_out.appendTo(undefinedCase, testNullCase);
1141                 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1142                 m_out.jump(continuation);
1143
1144                 m_out.appendTo(testNullCase, nullCase);
1145                 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1146                 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1147
1148                 m_out.appendTo(nullCase, testBooleanTrueCase);
1149                 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1150                 m_out.jump(continuation);
1151
1152                 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1153                 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1154                 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1155
1156                 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1157                 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1158                 m_out.jump(continuation);
1159
1160                 m_out.appendTo(convertBooleanFalseCase, continuation);
1161
1162                 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1163                 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse);
1164                 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1165                 m_out.jump(continuation);
1166
1167                 m_out.appendTo(continuation, lastNext);
1168                 setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1169                 return;
1170             }
1171             m_out.appendTo(nonDoubleCase, continuation);
1172             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1173             m_out.unreachable();
1174
1175             m_out.appendTo(continuation, lastNext);
1176
1177             setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble));
1178             return;
1179         }
1180             
1181         case Int52RepUse: {
1182             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1183             return;
1184         }
1185             
1186         default:
1187             DFG_CRASH(m_graph, m_node, "Bad use kind");
1188         }
1189     }
1190
1191     void compileDoubleAsInt32()
1192     {
1193         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1194         setInt32(integerValue);
1195     }
1196
1197     void compileValueRep()
1198     {
1199         switch (m_node->child1().useKind()) {
1200         case DoubleRepUse: {
1201             LValue value = lowDouble(m_node->child1());
1202             
1203             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1204                 value = m_out.select(
1205                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1206             }
1207             
1208             setJSValue(boxDouble(value));
1209             return;
1210         }
1211             
1212         case Int52RepUse: {
1213             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1214             return;
1215         }
1216             
1217         default:
1218             DFG_CRASH(m_graph, m_node, "Bad use kind");
1219         }
1220     }
1221     
1222     void compileInt52Rep()
1223     {
1224         switch (m_node->child1().useKind()) {
1225         case Int32Use:
1226             setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1())));
1227             return;
1228             
1229         case MachineIntUse:
1230             setStrictInt52(
1231                 jsValueToStrictInt52(
1232                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1233             return;
1234             
1235         case DoubleRepMachineIntUse:
1236             setStrictInt52(
1237                 doubleToStrictInt52(
1238                     m_node->child1(), lowDouble(m_node->child1())));
1239             return;
1240             
1241         default:
1242             RELEASE_ASSERT_NOT_REACHED();
1243         }
1244     }
1245     
1246     void compileValueToInt32()
1247     {
1248         switch (m_node->child1().useKind()) {
1249         case Int52RepUse:
1250             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1251             break;
1252             
1253         case DoubleRepUse:
1254             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1255             break;
1256             
1257         case NumberUse:
1258         case NotCellUse: {
1259             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1260             if (isValid(value)) {
1261                 setInt32(value.value());
1262                 break;
1263             }
1264             
1265             value = m_jsValueValues.get(m_node->child1().node());
1266             if (isValid(value)) {
1267                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1268                 break;
1269             }
1270             
1271             // We'll basically just get here for constants. But it's good to have this
1272             // catch-all since we often add new representations into the mix.
1273             setInt32(
1274                 numberOrNotCellToInt32(
1275                     m_node->child1(),
1276                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1277             break;
1278         }
1279             
1280         default:
1281             DFG_CRASH(m_graph, m_node, "Bad use kind");
1282             break;
1283         }
1284     }
1285     
1286     void compileBooleanToNumber()
1287     {
1288         switch (m_node->child1().useKind()) {
1289         case BooleanUse: {
1290             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1291             return;
1292         }
1293             
1294         case UntypedUse: {
1295             LValue value = lowJSValue(m_node->child1());
1296             
1297             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1298                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1299                 return;
1300             }
1301             
1302             LBasicBlock booleanCase = m_out.newBlock();
1303             LBasicBlock continuation = m_out.newBlock();
1304             
1305             ValueFromBlock notBooleanResult = m_out.anchor(value);
1306             m_out.branch(
1307                 isBoolean(value, provenType(m_node->child1())),
1308                 unsure(booleanCase), unsure(continuation));
1309             
1310             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1311             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1312                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1313             m_out.jump(continuation);
1314             
1315             m_out.appendTo(continuation, lastNext);
1316             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1317             return;
1318         }
1319             
1320         default:
1321             RELEASE_ASSERT_NOT_REACHED();
1322             return;
1323         }
1324     }
1325
1326     void compileExtractOSREntryLocal()
1327     {
1328         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1329             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1330         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1331     }
1332     
1333     void compileGetStack()
1334     {
1335         // GetLocals arise only for captured variables and arguments. For arguments, we might have
1336         // already loaded it.
1337         if (LValue value = m_loadedArgumentValues.get(m_node)) {
1338             setJSValue(value);
1339             return;
1340         }
1341         
1342         StackAccessData* data = m_node->stackAccessData();
1343         AbstractValue& value = m_state.variables().operand(data->local);
1344         
1345         DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1346         DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1347         
1348         if (isInt32Speculation(value.m_type))
1349             setInt32(m_out.load32(payloadFor(data->machineLocal)));
1350         else
1351             setJSValue(m_out.load64(addressFor(data->machineLocal)));
1352     }
1353     
1354     void compilePutStack()
1355     {
1356         StackAccessData* data = m_node->stackAccessData();
1357         switch (data->format) {
1358         case FlushedJSValue: {
1359             LValue value = lowJSValue(m_node->child1());
1360             m_out.store64(value, addressFor(data->machineLocal));
1361             break;
1362         }
1363             
1364         case FlushedDouble: {
1365             LValue value = lowDouble(m_node->child1());
1366             m_out.storeDouble(value, addressFor(data->machineLocal));
1367             break;
1368         }
1369             
1370         case FlushedInt32: {
1371             LValue value = lowInt32(m_node->child1());
1372             m_out.store32(value, payloadFor(data->machineLocal));
1373             break;
1374         }
1375             
1376         case FlushedInt52: {
1377             LValue value = lowInt52(m_node->child1());
1378             m_out.store64(value, addressFor(data->machineLocal));
1379             break;
1380         }
1381             
1382         case FlushedCell: {
1383             LValue value = lowCell(m_node->child1());
1384             m_out.store64(value, addressFor(data->machineLocal));
1385             break;
1386         }
1387             
1388         case FlushedBoolean: {
1389             speculateBoolean(m_node->child1());
1390             m_out.store64(
1391                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1392                 addressFor(data->machineLocal));
1393             break;
1394         }
1395             
1396         default:
1397             DFG_CRASH(m_graph, m_node, "Bad flush format");
1398             break;
1399         }
1400     }
1401     
1402     void compileNoOp()
1403     {
1404         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1405     }
1406     
1407     void compileToThis()
1408     {
1409         LValue value = lowJSValue(m_node->child1());
1410         
1411         LBasicBlock isCellCase = m_out.newBlock();
1412         LBasicBlock slowCase = m_out.newBlock();
1413         LBasicBlock continuation = m_out.newBlock();
1414         
1415         m_out.branch(
1416             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1417         
1418         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1419         ValueFromBlock fastResult = m_out.anchor(value);
1420         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1421         
1422         m_out.appendTo(slowCase, continuation);
1423         J_JITOperation_EJ function;
1424         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1425             function = operationToThisStrict;
1426         else
1427             function = operationToThis;
1428         ValueFromBlock slowResult = m_out.anchor(
1429             vmCall(m_out.int64, m_out.operation(function), m_callFrame, value));
1430         m_out.jump(continuation);
1431         
1432         m_out.appendTo(continuation, lastNext);
1433         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1434     }
1435     
1436     void compileValueAdd()
1437     {
1438         emitBinarySnippet<JITAddGenerator>(operationValueAdd);
1439     }
1440     
1441     void compileStrCat()
1442     {
1443         LValue result;
1444         if (m_node->child3()) {
1445             result = vmCall(
1446                 m_out.int64, m_out.operation(operationStrCat3), m_callFrame,
1447                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1448                 lowJSValue(m_node->child2(), ManualOperandSpeculation),
1449                 lowJSValue(m_node->child3(), ManualOperandSpeculation));
1450         } else {
1451             result = vmCall(
1452                 m_out.int64, m_out.operation(operationStrCat2), m_callFrame,
1453                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1454                 lowJSValue(m_node->child2(), ManualOperandSpeculation));
1455         }
1456         setJSValue(result);
1457     }
1458     
1459     void compileArithAddOrSub()
1460     {
1461         bool isSub =  m_node->op() == ArithSub;
1462         switch (m_node->binaryUseKind()) {
1463         case Int32Use: {
1464             LValue left = lowInt32(m_node->child1());
1465             LValue right = lowInt32(m_node->child2());
1466
1467             if (!shouldCheckOverflow(m_node->arithMode())) {
1468                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1469                 break;
1470             }
1471
1472             CheckValue* result =
1473                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
1474             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1475             setInt32(result);
1476             break;
1477         }
1478             
1479         case Int52RepUse: {
1480             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)
1481                 && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) {
1482                 Int52Kind kind;
1483                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1484                 LValue right = lowInt52(m_node->child2(), kind);
1485                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1486                 break;
1487             }
1488
1489             LValue left = lowInt52(m_node->child1());
1490             LValue right = lowInt52(m_node->child2());
1491             CheckValue* result =
1492                 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
1493             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1494             setInt52(result);
1495             break;
1496         }
1497             
1498         case DoubleRepUse: {
1499             LValue C1 = lowDouble(m_node->child1());
1500             LValue C2 = lowDouble(m_node->child2());
1501
1502             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1503             break;
1504         }
1505
1506         case UntypedUse: {
1507             if (!isSub) {
1508                 DFG_CRASH(m_graph, m_node, "Bad use kind");
1509                 break;
1510             }
1511
1512             emitBinarySnippet<JITSubGenerator>(operationValueSub);
1513             break;
1514         }
1515
1516         default:
1517             DFG_CRASH(m_graph, m_node, "Bad use kind");
1518             break;
1519         }
1520     }
1521
1522     void compileArithClz32()
1523     {
1524         LValue operand = lowInt32(m_node->child1());
1525         setInt32(m_out.ctlz32(operand));
1526     }
1527     
1528     void compileArithMul()
1529     {
1530         switch (m_node->binaryUseKind()) {
1531         case Int32Use: {
1532             LValue left = lowInt32(m_node->child1());
1533             LValue right = lowInt32(m_node->child2());
1534             
1535             LValue result;
1536
1537             if (!shouldCheckOverflow(m_node->arithMode()))
1538                 result = m_out.mul(left, right);
1539             else {
1540                 CheckValue* speculation = m_out.speculateMul(left, right);
1541                 blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin);
1542                 result = speculation;
1543             }
1544             
1545             if (shouldCheckNegativeZero(m_node->arithMode())) {
1546                 LBasicBlock slowCase = m_out.newBlock();
1547                 LBasicBlock continuation = m_out.newBlock();
1548                 
1549                 m_out.branch(
1550                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1551                 
1552                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1553                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero));
1554                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero));
1555                 m_out.jump(continuation);
1556                 m_out.appendTo(continuation, lastNext);
1557             }
1558             
1559             setInt32(result);
1560             break;
1561         }
1562             
1563         case Int52RepUse: {
1564             Int52Kind kind;
1565             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1566             LValue right = lowInt52(m_node->child2(), opposite(kind));
1567
1568             CheckValue* result = m_out.speculateMul(left, right);
1569             blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
1570
1571             if (shouldCheckNegativeZero(m_node->arithMode())) {
1572                 LBasicBlock slowCase = m_out.newBlock();
1573                 LBasicBlock continuation = m_out.newBlock();
1574                 
1575                 m_out.branch(
1576                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1577                 
1578                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1579                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero));
1580                 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero));
1581                 m_out.jump(continuation);
1582                 m_out.appendTo(continuation, lastNext);
1583             }
1584             
1585             setInt52(result);
1586             break;
1587         }
1588             
1589         case DoubleRepUse: {
1590             setDouble(
1591                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1592             break;
1593         }
1594
1595         case UntypedUse: {
1596             emitBinarySnippet<JITMulGenerator>(operationValueMul);
1597             break;
1598         }
1599
1600         default:
1601             DFG_CRASH(m_graph, m_node, "Bad use kind");
1602             break;
1603         }
1604     }
1605
1606     void compileArithDiv()
1607     {
1608         switch (m_node->binaryUseKind()) {
1609         case Int32Use: {
1610             LValue numerator = lowInt32(m_node->child1());
1611             LValue denominator = lowInt32(m_node->child2());
1612
1613             if (shouldCheckNegativeZero(m_node->arithMode())) {
1614                 LBasicBlock zeroNumerator = m_out.newBlock();
1615                 LBasicBlock numeratorContinuation = m_out.newBlock();
1616
1617                 m_out.branch(
1618                     m_out.isZero32(numerator),
1619                     rarely(zeroNumerator), usually(numeratorContinuation));
1620
1621                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1622
1623                 speculate(
1624                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1625
1626                 m_out.jump(numeratorContinuation);
1627
1628                 m_out.appendTo(numeratorContinuation, innerLastNext);
1629             }
1630             
1631             if (shouldCheckOverflow(m_node->arithMode())) {
1632                 LBasicBlock unsafeDenominator = m_out.newBlock();
1633                 LBasicBlock continuation = m_out.newBlock();
1634
1635                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1636                 m_out.branch(
1637                     m_out.above(adjustedDenominator, m_out.int32One),
1638                     usually(continuation), rarely(unsafeDenominator));
1639
1640                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1641                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1642                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
1643                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
1644                 m_out.jump(continuation);
1645
1646                 m_out.appendTo(continuation, lastNext);
1647                 LValue result = m_out.div(numerator, denominator);
1648                 speculate(
1649                     Overflow, noValue(), 0,
1650                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1651                 setInt32(result);
1652             } else
1653                 setInt32(m_out.chillDiv(numerator, denominator));
1654
1655             break;
1656         }
1657             
1658         case DoubleRepUse: {
1659             setDouble(m_out.doubleDiv(
1660                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1661             break;
1662         }
1663
1664         case UntypedUse: {
1665             emitBinarySnippet<JITDivGenerator, NeedScratchFPR>(operationValueDiv);
1666             break;
1667         }
1668
1669         default:
1670             DFG_CRASH(m_graph, m_node, "Bad use kind");
1671             break;
1672         }
1673     }
1674     
1675     void compileArithMod()
1676     {
1677         switch (m_node->binaryUseKind()) {
1678         case Int32Use: {
1679             LValue numerator = lowInt32(m_node->child1());
1680             LValue denominator = lowInt32(m_node->child2());
1681
1682             LValue remainder;
1683             if (shouldCheckOverflow(m_node->arithMode())) {
1684                 LBasicBlock unsafeDenominator = m_out.newBlock();
1685                 LBasicBlock continuation = m_out.newBlock();
1686
1687                 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1688                 m_out.branch(
1689                     m_out.above(adjustedDenominator, m_out.int32One),
1690                     usually(continuation), rarely(unsafeDenominator));
1691
1692                 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1693                 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1694                 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
1695                 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
1696                 m_out.jump(continuation);
1697
1698                 m_out.appendTo(continuation, lastNext);
1699                 LValue result = m_out.mod(numerator, denominator);
1700                 remainder = result;
1701             } else
1702                 remainder = m_out.chillMod(numerator, denominator);
1703
1704             if (shouldCheckNegativeZero(m_node->arithMode())) {
1705                 LBasicBlock negativeNumerator = m_out.newBlock();
1706                 LBasicBlock numeratorContinuation = m_out.newBlock();
1707
1708                 m_out.branch(
1709                     m_out.lessThan(numerator, m_out.int32Zero),
1710                     unsure(negativeNumerator), unsure(numeratorContinuation));
1711
1712                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1713
1714                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1715
1716                 m_out.jump(numeratorContinuation);
1717
1718                 m_out.appendTo(numeratorContinuation, innerLastNext);
1719             }
1720
1721             setInt32(remainder);
1722             break;
1723         }
1724             
1725         case DoubleRepUse: {
1726             setDouble(
1727                 m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1728             break;
1729         }
1730             
1731         default:
1732             DFG_CRASH(m_graph, m_node, "Bad use kind");
1733             break;
1734         }
1735     }
1736
1737     void compileArithMinOrMax()
1738     {
1739         switch (m_node->binaryUseKind()) {
1740         case Int32Use: {
1741             LValue left = lowInt32(m_node->child1());
1742             LValue right = lowInt32(m_node->child2());
1743             
1744             setInt32(
1745                 m_out.select(
1746                     m_node->op() == ArithMin
1747                         ? m_out.lessThan(left, right)
1748                         : m_out.lessThan(right, left),
1749                     left, right));
1750             break;
1751         }
1752             
1753         case DoubleRepUse: {
1754             LValue left = lowDouble(m_node->child1());
1755             LValue right = lowDouble(m_node->child2());
1756             
1757             LBasicBlock notLessThan = m_out.newBlock();
1758             LBasicBlock continuation = m_out.newBlock();
1759             
1760             Vector<ValueFromBlock, 2> results;
1761             
1762             results.append(m_out.anchor(left));
1763             m_out.branch(
1764                 m_node->op() == ArithMin
1765                     ? m_out.doubleLessThan(left, right)
1766                     : m_out.doubleGreaterThan(left, right),
1767                 unsure(continuation), unsure(notLessThan));
1768             
1769             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1770             results.append(m_out.anchor(m_out.select(
1771                 m_node->op() == ArithMin
1772                     ? m_out.doubleGreaterThanOrEqual(left, right)
1773                     : m_out.doubleLessThanOrEqual(left, right),
1774                 right, m_out.constDouble(PNaN))));
1775             m_out.jump(continuation);
1776             
1777             m_out.appendTo(continuation, lastNext);
1778             setDouble(m_out.phi(m_out.doubleType, results));
1779             break;
1780         }
1781             
1782         default:
1783             DFG_CRASH(m_graph, m_node, "Bad use kind");
1784             break;
1785         }
1786     }
1787     
1788     void compileArithAbs()
1789     {
1790         switch (m_node->child1().useKind()) {
1791         case Int32Use: {
1792             LValue value = lowInt32(m_node->child1());
1793
1794             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1795             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1796
1797             if (shouldCheckOverflow(m_node->arithMode()))
1798                 speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
1799
1800             setInt32(result);
1801             break;
1802         }
1803             
1804         case DoubleRepUse: {
1805             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1806             break;
1807         }
1808             
1809         default:
1810             DFG_CRASH(m_graph, m_node, "Bad use kind");
1811             break;
1812         }
1813     }
1814
1815     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1816
1817     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1818
1819     void compileArithPow()
1820     {
1821         if (m_node->child2().useKind() == Int32Use)
1822             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1823         else {
1824             LValue base = lowDouble(m_node->child1());
1825             LValue exponent = lowDouble(m_node->child2());
1826
1827             LBasicBlock integerExponentIsSmallBlock = m_out.newBlock();
1828             LBasicBlock integerExponentPowBlock = m_out.newBlock();
1829             LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock();
1830             LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock();
1831             LBasicBlock nanExceptionBaseIsOne = m_out.newBlock();
1832             LBasicBlock powBlock = m_out.newBlock();
1833             LBasicBlock nanExceptionResultIsNaN = m_out.newBlock();
1834             LBasicBlock continuation = m_out.newBlock();
1835
1836             LValue integerExponent = m_out.doubleToInt(exponent);
1837             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1838             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1839             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1840
1841             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1842             LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1843             m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1844
1845             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1846             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1847             m_out.jump(continuation);
1848
1849             // If y is NaN, the result is NaN.
1850             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1851             LValue exponentIsNaN;
1852             if (provenType(m_node->child2()) & SpecDoubleNaN)
1853                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1854             else
1855                 exponentIsNaN = m_out.booleanFalse;
1856             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1857
1858             // If abs(x) is 1 and y is +infinity, the result is NaN.
1859             // If abs(x) is 1 and y is -infinity, the result is NaN.
1860             m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1861             LValue absoluteExponent = m_out.doubleAbs(exponent);
1862             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1863             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1864
1865             m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1866             LValue absoluteBase = m_out.doubleAbs(base);
1867             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1868             m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1869
1870             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1871             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1872             m_out.jump(continuation);
1873
1874             m_out.appendTo(nanExceptionResultIsNaN, continuation);
1875             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1876             m_out.jump(continuation);
1877
1878             m_out.appendTo(continuation, lastNext);
1879             setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1880         }
1881     }
1882
1883     void compileArithRandom()
1884     {
1885         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
1886
1887         // Inlined WeakRandom::advance().
1888         // uint64_t x = m_low;
1889         void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
1890         LValue low = m_out.load64(m_out.absolute(lowAddress));
1891         // uint64_t y = m_high;
1892         void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();
1893         LValue high = m_out.load64(m_out.absolute(highAddress));
1894         // m_low = y;
1895         m_out.store64(high, m_out.absolute(lowAddress));
1896
1897         // x ^= x << 23;
1898         LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low);
1899
1900         // x ^= x >> 17;
1901         LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1);
1902
1903         // x ^= y ^ (y >> 26);
1904         LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2);
1905
1906         // m_high = x;
1907         m_out.store64(phase3, m_out.absolute(highAddress));
1908
1909         // return x + y;
1910         LValue random64 = m_out.add(phase3, high);
1911
1912         // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
1913         LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1));
1914
1915         LValue double53Integer = m_out.intToDouble(random53);
1916
1917         // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
1918         // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
1919         static const double scale = 1.0 / (1ULL << 53);
1920
1921         // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
1922         // It just reduces the exp part of the given 53bit double integer.
1923         // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
1924         // Now we get 53bit precision random double value in [0, 1).
1925         LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale));
1926
1927         setDouble(result);
1928     }
1929
1930     void compileArithRound()
1931     {
1932         LValue result = nullptr;
1933
1934         if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) {
1935             LValue value = lowDouble(m_node->child1());
1936             result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5)));
1937         } else {
1938             LBasicBlock realPartIsMoreThanHalf = m_out.newBlock();
1939             LBasicBlock continuation = m_out.newBlock();
1940
1941             LValue value = lowDouble(m_node->child1());
1942             LValue integerValue = m_out.doubleCeil(value);
1943             ValueFromBlock integerValueResult = m_out.anchor(integerValue);
1944
1945             LValue realPart = m_out.doubleSub(integerValue, value);
1946
1947             m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
1948
1949             LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
1950             LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
1951             ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
1952             m_out.jump(continuation);
1953             m_out.appendTo(continuation, lastNext);
1954
1955             result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult);
1956         }
1957
1958         if (producesInteger(m_node->arithRoundingMode())) {
1959             LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
1960             setInt32(integerValue);
1961         } else
1962             setDouble(result);
1963     }
1964
1965     void compileArithFloor()
1966     {
1967         LValue value = lowDouble(m_node->child1());
1968         LValue integerValue = m_out.doubleFloor(value);
1969         if (producesInteger(m_node->arithRoundingMode()))
1970             setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
1971         else
1972             setDouble(integerValue);
1973     }
1974
1975     void compileArithCeil()
1976     {
1977         LValue value = lowDouble(m_node->child1());
1978         LValue integerValue = m_out.doubleCeil(value);
1979         if (producesInteger(m_node->arithRoundingMode()))
1980             setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
1981         else
1982             setDouble(integerValue);
1983     }
1984
1985     void compileArithTrunc()
1986     {
1987         LValue value = lowDouble(m_node->child1());
1988         LValue result = m_out.doubleTrunc(value);
1989         if (producesInteger(m_node->arithRoundingMode()))
1990             setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())));
1991         else
1992             setDouble(result);
1993     }
1994
1995     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1996
1997     void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
1998     
1999     void compileArithFRound()
2000     {
2001         setDouble(m_out.fround(lowDouble(m_node->child1())));
2002     }
2003     
2004     void compileArithNegate()
2005     {
2006         switch (m_node->child1().useKind()) {
2007         case Int32Use: {
2008             LValue value = lowInt32(m_node->child1());
2009             
2010             LValue result;
2011             if (!shouldCheckOverflow(m_node->arithMode()))
2012                 result = m_out.neg(value);
2013             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
2014                 CheckValue* check = m_out.speculateSub(m_out.int32Zero, value);
2015                 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
2016                 result = check;
2017             } else {
2018                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
2019                 result = m_out.neg(value);
2020             }
2021
2022             setInt32(result);
2023             break;
2024         }
2025             
2026         case Int52RepUse: {
2027             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) {
2028                 Int52Kind kind;
2029                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
2030                 LValue result = m_out.neg(value);
2031                 if (shouldCheckNegativeZero(m_node->arithMode()))
2032                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
2033                 setInt52(result, kind);
2034                 break;
2035             }
2036             
2037             LValue value = lowInt52(m_node->child1());
2038             CheckValue* result = m_out.speculateSub(m_out.int64Zero, value);
2039             blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin);
2040             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
2041             setInt52(result);
2042             break;
2043         }
2044             
2045         case DoubleRepUse: {
2046             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
2047             break;
2048         }
2049             
2050         default:
2051             DFG_CRASH(m_graph, m_node, "Bad use kind");
2052             break;
2053         }
2054     }
2055     
2056     void compileBitAnd()
2057     {
2058         if (m_node->isBinaryUseKind(UntypedUse)) {
2059             emitBinaryBitOpSnippet<JITBitAndGenerator>(operationValueBitAnd);
2060             return;
2061         }
2062         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2063     }
2064     
2065     void compileBitOr()
2066     {
2067         if (m_node->isBinaryUseKind(UntypedUse)) {
2068             emitBinaryBitOpSnippet<JITBitOrGenerator>(operationValueBitOr);
2069             return;
2070         }
2071         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2072     }
2073     
2074     void compileBitXor()
2075     {
2076         if (m_node->isBinaryUseKind(UntypedUse)) {
2077             emitBinaryBitOpSnippet<JITBitXorGenerator>(operationValueBitXor);
2078             return;
2079         }
2080         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2081     }
2082     
2083     void compileBitRShift()
2084     {
2085         if (m_node->isBinaryUseKind(UntypedUse)) {
2086             emitRightShiftSnippet(JITRightShiftGenerator::SignedShift);
2087             return;
2088         }
2089         setInt32(m_out.aShr(
2090             lowInt32(m_node->child1()),
2091             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2092     }
2093     
2094     void compileBitLShift()
2095     {
2096         if (m_node->isBinaryUseKind(UntypedUse)) {
2097             emitBinaryBitOpSnippet<JITLeftShiftGenerator>(operationValueBitLShift);
2098             return;
2099         }
2100         setInt32(m_out.shl(
2101             lowInt32(m_node->child1()),
2102             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2103     }
2104     
2105     void compileBitURShift()
2106     {
2107         if (m_node->isBinaryUseKind(UntypedUse)) {
2108             emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift);
2109             return;
2110         }
2111         setInt32(m_out.lShr(
2112             lowInt32(m_node->child1()),
2113             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2114     }
2115     
2116     void compileUInt32ToNumber()
2117     {
2118         LValue value = lowInt32(m_node->child1());
2119
2120         if (doesOverflow(m_node->arithMode())) {
2121             setDouble(m_out.unsignedToDouble(value));
2122             return;
2123         }
2124         
2125         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
2126         setInt32(value);
2127     }
2128     
2129     void compileCheckStructure()
2130     {
2131         ExitKind exitKind;
2132         if (m_node->child1()->hasConstant())
2133             exitKind = BadConstantCache;
2134         else
2135             exitKind = BadCache;
2136
2137         switch (m_node->child1().useKind()) {
2138         case CellUse:
2139         case KnownCellUse: {
2140             LValue cell = lowCell(m_node->child1());
2141             
2142             checkStructure(
2143                 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
2144                 exitKind, m_node->structureSet(),
2145                 [&] (Structure* structure) {
2146                     return weakStructureID(structure);
2147                 });
2148             return;
2149         }
2150
2151         case CellOrOtherUse: {
2152             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
2153
2154             LBasicBlock cellCase = m_out.newBlock();
2155             LBasicBlock notCellCase = m_out.newBlock();
2156             LBasicBlock continuation = m_out.newBlock();
2157
2158             m_out.branch(
2159                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2160
2161             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2162             checkStructure(
2163                 m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value),
2164                 exitKind, m_node->structureSet(),
2165                 [&] (Structure* structure) {
2166                     return weakStructureID(structure);
2167                 });
2168             m_out.jump(continuation);
2169
2170             m_out.appendTo(notCellCase, continuation);
2171             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value));
2172             m_out.jump(continuation);
2173
2174             m_out.appendTo(continuation, lastNext);
2175             return;
2176         }
2177
2178         default:
2179             DFG_CRASH(m_graph, m_node, "Bad use kind");
2180             return;
2181         }
2182     }
2183     
2184     void compileCheckCell()
2185     {
2186         LValue cell = lowCell(m_node->child1());
2187         
2188         speculate(
2189             BadCell, jsValueValue(cell), m_node->child1().node(),
2190             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
2191     }
2192     
2193     void compileCheckBadCell()
2194     {
2195         terminate(BadCell);
2196     }
2197
2198     void compileCheckNotEmpty()
2199     {
2200         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
2201     }
2202
2203     void compileCheckIdent()
2204     {
2205         UniquedStringImpl* uid = m_node->uidOperand();
2206         if (uid->isSymbol()) {
2207             LValue symbol = lowSymbol(m_node->child1());
2208             LValue stringImpl = m_out.loadPtr(symbol, m_heaps.Symbol_privateName);
2209             speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
2210         } else {
2211             LValue string = lowStringIdent(m_node->child1());
2212             LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
2213             speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
2214         }
2215     }
2216
2217     void compileGetExecutable()
2218     {
2219         LValue cell = lowCell(m_node->child1());
2220         speculateFunction(m_node->child1(), cell);
2221         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
2222     }
2223     
2224     void compileArrayifyToStructure()
2225     {
2226         LValue cell = lowCell(m_node->child1());
2227         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
2228         
2229         LBasicBlock unexpectedStructure = m_out.newBlock();
2230         LBasicBlock continuation = m_out.newBlock();
2231         
2232         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2233         
2234         m_out.branch(
2235             m_out.notEqual(structureID, weakStructureID(m_node->structure())),
2236             rarely(unexpectedStructure), usually(continuation));
2237         
2238         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
2239         
2240         if (property) {
2241             switch (m_node->arrayMode().type()) {
2242             case Array::Int32:
2243             case Array::Double:
2244             case Array::Contiguous:
2245                 speculate(
2246                     Uncountable, noValue(), 0,
2247                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
2248                 break;
2249             default:
2250                 break;
2251             }
2252         }
2253         
2254         switch (m_node->arrayMode().type()) {
2255         case Array::Int32:
2256             vmCall(m_out.voidType, m_out.operation(operationEnsureInt32), m_callFrame, cell);
2257             break;
2258         case Array::Double:
2259             vmCall(m_out.voidType, m_out.operation(operationEnsureDouble), m_callFrame, cell);
2260             break;
2261         case Array::Contiguous:
2262             vmCall(m_out.voidType, m_out.operation(operationEnsureContiguous), m_callFrame, cell);
2263             break;
2264         case Array::ArrayStorage:
2265         case Array::SlowPutArrayStorage:
2266             vmCall(m_out.voidType, m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
2267             break;
2268         default:
2269             DFG_CRASH(m_graph, m_node, "Bad array type");
2270             break;
2271         }
2272         
2273         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2274         speculate(
2275             BadIndexingType, jsValueValue(cell), 0,
2276             m_out.notEqual(structureID, weakStructureID(m_node->structure())));
2277         m_out.jump(continuation);
2278         
2279         m_out.appendTo(continuation, lastNext);
2280     }
2281     
2282     void compilePutStructure()
2283     {
2284         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
2285
2286         Structure* oldStructure = m_node->transition()->previous;
2287         Structure* newStructure = m_node->transition()->next;
2288         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
2289         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
2290         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
2291
2292         LValue cell = lowCell(m_node->child1()); 
2293         m_out.store32(
2294             weakStructureID(newStructure),
2295             cell, m_heaps.JSCell_structureID);
2296     }
2297     
2298     void compileGetById()
2299     {
2300         switch (m_node->child1().useKind()) {
2301         case CellUse: {
2302             setJSValue(getById(lowCell(m_node->child1())));
2303             return;
2304         }
2305             
2306         case UntypedUse: {
2307             // This is pretty weird, since we duplicate the slow path both here and in the
2308             // code generated by the IC. We should investigate making this less bad.
2309             // https://bugs.webkit.org/show_bug.cgi?id=127830
2310             LValue value = lowJSValue(m_node->child1());
2311             
2312             LBasicBlock cellCase = m_out.newBlock();
2313             LBasicBlock notCellCase = m_out.newBlock();
2314             LBasicBlock continuation = m_out.newBlock();
2315             
2316             m_out.branch(
2317                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2318             
2319             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2320             ValueFromBlock cellResult = m_out.anchor(getById(value));
2321             m_out.jump(continuation);
2322             
2323             m_out.appendTo(notCellCase, continuation);
2324             ValueFromBlock notCellResult = m_out.anchor(vmCall(
2325                 m_out.int64, m_out.operation(operationGetByIdGeneric),
2326                 m_callFrame, value,
2327                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2328             m_out.jump(continuation);
2329             
2330             m_out.appendTo(continuation, lastNext);
2331             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2332             return;
2333         }
2334             
2335         default:
2336             DFG_CRASH(m_graph, m_node, "Bad use kind");
2337             return;
2338         }
2339     }
2340     
2341     void compilePutById()
2342     {
2343         Node* node = m_node;
2344         
2345         // See above; CellUse is easier so we do only that for now.
2346         ASSERT(node->child1().useKind() == CellUse);
2347
2348         LValue base = lowCell(node->child1());
2349         LValue value = lowJSValue(node->child2());
2350         auto uid = m_graph.identifiers()[node->identifierNumber()];
2351
2352         B3::PatchpointValue* patchpoint = m_out.patchpoint(Void);
2353         patchpoint->appendSomeRegister(base);
2354         patchpoint->appendSomeRegister(value);
2355         patchpoint->clobber(RegisterSet::macroScratchRegisters());
2356
2357         // FIXME: If this is a PutByIdFlush, we might want to late-clobber volatile registers.
2358         // https://bugs.webkit.org/show_bug.cgi?id=152848
2359
2360         RefPtr<PatchpointExceptionHandle> exceptionHandle =
2361             preparePatchpointForExceptions(patchpoint);
2362
2363         State* state = &m_ftlState;
2364         ECMAMode ecmaMode = m_graph.executableFor(node->origin.semantic)->ecmaMode();
2365         
2366         patchpoint->setGenerator(
2367             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2368                 AllowMacroScratchRegisterUsage allowScratch(jit);
2369
2370                 CallSiteIndex callSiteIndex =
2371                     state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
2372
2373                 Box<CCallHelpers::JumpList> exceptions =
2374                     exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2375
2376                 // JS setter call ICs generated by the PutById IC will need this.
2377                 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
2378
2379                 auto generator = Box<JITPutByIdGenerator>::create(
2380                     jit.codeBlock(), node->origin.semantic, callSiteIndex,
2381                     params.unavailableRegisters(), JSValueRegs(params[0].gpr()),
2382                     JSValueRegs(params[1].gpr()), GPRInfo::patchpointScratchRegister, ecmaMode,
2383                     node->op() == PutByIdDirect ? Direct : NotDirect);
2384
2385                 generator->generateFastPath(jit);
2386                 CCallHelpers::Label done = jit.label();
2387
2388                 params.addLatePath(
2389                     [=] (CCallHelpers& jit) {
2390                         AllowMacroScratchRegisterUsage allowScratch(jit);
2391
2392                         generator->slowPathJump().link(&jit);
2393                         CCallHelpers::Label slowPathBegin = jit.label();
2394                         CCallHelpers::Call slowPathCall = callOperation(
2395                             *state, params.unavailableRegisters(), jit, node->origin.semantic,
2396                             exceptions.get(), generator->slowPathFunction(), InvalidGPRReg,
2397                             CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
2398                             params[0].gpr(), CCallHelpers::TrustedImmPtr(uid)).call();
2399                         jit.jump().linkTo(done, &jit);
2400
2401                         generator->reportSlowPathCall(slowPathBegin, slowPathCall);
2402
2403                         jit.addLinkTask(
2404                             [=] (LinkBuffer& linkBuffer) {
2405                                 generator->finalize(linkBuffer);
2406                             });
2407                     });
2408             });
2409     }
2410     
2411     void compileGetButterfly()
2412     {
2413         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2414     }
2415
2416     void compileConstantStoragePointer()
2417     {
2418         setStorage(m_out.constIntPtr(m_node->storagePointer()));
2419     }
2420     
2421     void compileGetIndexedPropertyStorage()
2422     {
2423         LValue cell = lowCell(m_node->child1());
2424         
2425         if (m_node->arrayMode().type() == Array::String) {
2426             LBasicBlock slowPath = m_out.newBlock();
2427             LBasicBlock continuation = m_out.newBlock();
2428
2429             LValue fastResultValue = m_out.loadPtr(cell, m_heaps.JSString_value);
2430             ValueFromBlock fastResult = m_out.anchor(fastResultValue);
2431             
2432             m_out.branch(
2433                 m_out.notNull(fastResultValue), usually(continuation), rarely(slowPath));
2434             
2435             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2436             
2437             ValueFromBlock slowResult = m_out.anchor(
2438                 vmCall(m_out.intPtr, m_out.operation(operationResolveRope), m_callFrame, cell));
2439             
2440             m_out.jump(continuation);
2441             
2442             m_out.appendTo(continuation, lastNext);
2443             
2444             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2445             return;
2446         }
2447         
2448         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2449     }
2450     
2451     void compileCheckArray()
2452     {
2453         Edge edge = m_node->child1();
2454         LValue cell = lowCell(edge);
2455         
2456         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
2457             return;
2458         
2459         speculate(
2460             BadIndexingType, jsValueValue(cell), 0,
2461             m_out.logicalNot(isArrayType(cell, m_node->arrayMode())));
2462     }
2463
2464     void compileGetTypedArrayByteOffset()
2465     {
2466         LValue basePtr = lowCell(m_node->child1());    
2467
2468         LBasicBlock simpleCase = m_out.newBlock();
2469         LBasicBlock wastefulCase = m_out.newBlock();
2470         LBasicBlock continuation = m_out.newBlock();
2471         
2472         LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2473         m_out.branch(
2474             m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2475             unsure(simpleCase), unsure(wastefulCase));
2476
2477         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2478
2479         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2480
2481         m_out.jump(continuation);
2482
2483         m_out.appendTo(wastefulCase, continuation);
2484
2485         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2486         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2487         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2488         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2489
2490         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2491
2492         m_out.jump(continuation);
2493         m_out.appendTo(continuation, lastNext);
2494
2495         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2496     }
2497     
2498     void compileGetArrayLength()
2499     {
2500         switch (m_node->arrayMode().type()) {
2501         case Array::Int32:
2502         case Array::Double:
2503         case Array::Contiguous: {
2504             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2505             return;
2506         }
2507             
2508         case Array::String: {
2509             LValue string = lowCell(m_node->child1());
2510             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2511             return;
2512         }
2513             
2514         case Array::DirectArguments: {
2515             LValue arguments = lowCell(m_node->child1());
2516             speculate(
2517                 ExoticObjectMode, noValue(), nullptr,
2518                 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2519             setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2520             return;
2521         }
2522             
2523         case Array::ScopedArguments: {
2524             LValue arguments = lowCell(m_node->child1());
2525             speculate(
2526                 ExoticObjectMode, noValue(), nullptr,
2527                 m_out.notZero32(m_out.load8ZeroExt32(arguments, m_heaps.ScopedArguments_overrodeThings)));
2528             setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2529             return;
2530         }
2531             
2532         default:
2533             if (m_node->arrayMode().isSomeTypedArrayView()) {
2534                 setInt32(
2535                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2536                 return;
2537             }
2538             
2539             DFG_CRASH(m_graph, m_node, "Bad array type");
2540             return;
2541         }
2542     }
2543     
2544     void compileCheckInBounds()
2545     {
2546         speculate(
2547             OutOfBounds, noValue(), 0,
2548             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2549     }
2550     
2551     void compileGetByVal()
2552     {
2553         switch (m_node->arrayMode().type()) {
2554         case Array::Int32:
2555         case Array::Contiguous: {
2556             LValue index = lowInt32(m_node->child2());
2557             LValue storage = lowStorage(m_node->child3());
2558             
2559             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2560                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2561             
2562             if (m_node->arrayMode().isInBounds()) {
2563                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2564                 LValue isHole = m_out.isZero64(result);
2565                 if (m_node->arrayMode().isSaneChain()) {
2566                     DFG_ASSERT(
2567                         m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous);
2568                     result = m_out.select(
2569                         isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
2570                 } else
2571                     speculate(LoadFromHole, noValue(), 0, isHole);
2572                 setJSValue(result);
2573                 return;
2574             }
2575             
2576             LValue base = lowCell(m_node->child1());
2577             
2578             LBasicBlock fastCase = m_out.newBlock();
2579             LBasicBlock slowCase = m_out.newBlock();
2580             LBasicBlock continuation = m_out.newBlock();
2581             
2582             m_out.branch(
2583                 m_out.aboveOrEqual(
2584                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2585                 rarely(slowCase), usually(fastCase));
2586             
2587             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2588
2589             LValue fastResultValue = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2590             ValueFromBlock fastResult = m_out.anchor(fastResultValue);
2591             m_out.branch(
2592                 m_out.isZero64(fastResultValue), rarely(slowCase), usually(continuation));
2593             
2594             m_out.appendTo(slowCase, continuation);
2595             ValueFromBlock slowResult = m_out.anchor(
2596                 vmCall(m_out.int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2597             m_out.jump(continuation);
2598             
2599             m_out.appendTo(continuation, lastNext);
2600             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2601             return;
2602         }
2603             
2604         case Array::Double: {
2605             LValue index = lowInt32(m_node->child2());
2606             LValue storage = lowStorage(m_node->child3());
2607             
2608             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2609             
2610             if (m_node->arrayMode().isInBounds()) {
2611                 LValue result = m_out.loadDouble(
2612                     baseIndex(heap, storage, index, m_node->child2()));
2613                 
2614                 if (!m_node->arrayMode().isSaneChain()) {
2615                     speculate(
2616                         LoadFromHole, noValue(), 0,
2617                         m_out.doubleNotEqualOrUnordered(result, result));
2618                 }
2619                 setDouble(result);
2620                 break;
2621             }
2622             
2623             LValue base = lowCell(m_node->child1());
2624             
2625             LBasicBlock inBounds = m_out.newBlock();
2626             LBasicBlock boxPath = m_out.newBlock();
2627             LBasicBlock slowCase = m_out.newBlock();
2628             LBasicBlock continuation = m_out.newBlock();
2629             
2630             m_out.branch(
2631                 m_out.aboveOrEqual(
2632                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2633                 rarely(slowCase), usually(inBounds));
2634             
2635             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2636             LValue doubleValue = m_out.loadDouble(
2637                 baseIndex(heap, storage, index, m_node->child2()));
2638             m_out.branch(
2639                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2640                 rarely(slowCase), usually(boxPath));
2641             
2642             m_out.appendTo(boxPath, slowCase);
2643             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2644             m_out.jump(continuation);
2645             
2646             m_out.appendTo(slowCase, continuation);
2647             ValueFromBlock slowResult = m_out.anchor(
2648                 vmCall(m_out.int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2649             m_out.jump(continuation);
2650             
2651             m_out.appendTo(continuation, lastNext);
2652             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2653             return;
2654         }
2655
2656         case Array::Undecided: {
2657             LValue index = lowInt32(m_node->child2());
2658
2659             speculate(OutOfBounds, noValue(), m_node, m_out.lessThan(index, m_out.int32Zero));
2660             setJSValue(m_out.constInt64(ValueUndefined));
2661             return;
2662         }
2663             
2664         case Array::DirectArguments: {
2665             LValue base = lowCell(m_node->child1());
2666             LValue index = lowInt32(m_node->child2());
2667             
2668             speculate(
2669                 ExoticObjectMode, noValue(), nullptr,
2670                 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2671             speculate(
2672                 ExoticObjectMode, noValue(), nullptr,
2673                 m_out.aboveOrEqual(
2674                     index,
2675                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2676
2677             TypedPointer address = m_out.baseIndex(
2678                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2679             setJSValue(m_out.load64(address));
2680             return;
2681         }
2682             
2683         case Array::ScopedArguments: {
2684             LValue base = lowCell(m_node->child1());
2685             LValue index = lowInt32(m_node->child2());
2686             
2687             speculate(
2688                 ExoticObjectMode, noValue(), nullptr,
2689                 m_out.aboveOrEqual(
2690                     index,
2691                     m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2692             
2693             LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2694             LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2695             
2696             LBasicBlock namedCase = m_out.newBlock();
2697             LBasicBlock overflowCase = m_out.newBlock();
2698             LBasicBlock continuation = m_out.newBlock();
2699             
2700             m_out.branch(
2701                 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2702             
2703             LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2704             
2705             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2706             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2707             
2708             TypedPointer address = m_out.baseIndex(
2709                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2710             LValue scopeOffset = m_out.load32(address);
2711             
2712             speculate(
2713                 ExoticObjectMode, noValue(), nullptr,
2714                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2715             
2716             address = m_out.baseIndex(
2717                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2718             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2719             m_out.jump(continuation);
2720             
2721             m_out.appendTo(overflowCase, continuation);
2722             
2723             address = m_out.baseIndex(
2724                 m_heaps.ScopedArguments_overflowStorage, base,
2725                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2726             LValue overflowValue = m_out.load64(address);
2727             speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2728             ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2729             m_out.jump(continuation);
2730             
2731             m_out.appendTo(continuation, lastNext);
2732             setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2733             return;
2734         }
2735             
2736         case Array::Generic: {
2737             setJSValue(vmCall(
2738                 m_out.int64, m_out.operation(operationGetByVal), m_callFrame,
2739                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2740             return;
2741         }
2742             
2743         case Array::String: {
2744             compileStringCharAt();
2745             return;
2746         }
2747             
2748         default: {
2749             LValue index = lowInt32(m_node->child2());
2750             LValue storage = lowStorage(m_node->child3());
2751             
2752             TypedArrayType type = m_node->arrayMode().typedArrayType();
2753             
2754             if (isTypedView(type)) {
2755                 TypedPointer pointer = TypedPointer(
2756                     m_heaps.typedArrayProperties,
2757                     m_out.add(
2758                         storage,
2759                         m_out.shl(
2760                             m_out.zeroExtPtr(index),
2761                             m_out.constIntPtr(logElementSize(type)))));
2762                 
2763                 if (isInt(type)) {
2764                     LValue result;
2765                     switch (elementSize(type)) {
2766                     case 1:
2767                         result = isSigned(type) ? m_out.load8SignExt32(pointer) :  m_out.load8ZeroExt32(pointer);
2768                         break;
2769                     case 2:
2770                         result = isSigned(type) ? m_out.load16SignExt32(pointer) :  m_out.load16ZeroExt32(pointer);
2771                         break;
2772                     case 4:
2773                         result = m_out.load32(pointer);
2774                         break;
2775                     default:
2776                         DFG_CRASH(m_graph, m_node, "Bad element size");
2777                     }
2778                     
2779                     if (elementSize(type) < 4 || isSigned(type)) {
2780                         setInt32(result);
2781                         return;
2782                     }
2783
2784                     if (m_node->shouldSpeculateInt32()) {
2785                         speculate(
2786                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2787                         setInt32(result);
2788                         return;
2789                     }
2790                     
2791                     if (m_node->shouldSpeculateMachineInt()) {
2792                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2793                         return;
2794                     }
2795                     
2796                     setDouble(m_out.unsignedToDouble(result));
2797                     return;
2798                 }
2799             
2800                 ASSERT(isFloat(type));
2801                 
2802                 LValue result;
2803                 switch (type) {
2804                 case TypeFloat32:
2805                     result = m_out.floatToDouble(m_out.loadFloat(pointer));
2806                     break;
2807                 case TypeFloat64:
2808                     result = m_out.loadDouble(pointer);
2809                     break;
2810                 default:
2811                     DFG_CRASH(m_graph, m_node, "Bad typed array type");
2812                 }
2813                 
2814                 setDouble(result);
2815                 return;
2816             }
2817             
2818             DFG_CRASH(m_graph, m_node, "Bad array type");
2819             return;
2820         } }
2821     }
2822     
2823     void compileGetMyArgumentByVal()
2824     {
2825         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2826         
2827         LValue index = lowInt32(m_node->child2());
2828         
2829         LValue limit;
2830         if (inlineCallFrame && !inlineCallFrame->isVarargs())
2831             limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2832         else {
2833             VirtualRegister argumentCountRegister;
2834             if (!inlineCallFrame)
2835                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2836             else
2837                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2838             limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2839         }
2840         
2841         speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2842         
2843         TypedPointer base;
2844         if (inlineCallFrame) {
2845             if (inlineCallFrame->arguments.size() <= 1) {
2846                 // We should have already exited due to the bounds check, above. Just tell the
2847                 // compiler that anything dominated by this instruction is not reachable, so
2848                 // that we don't waste time generating such code. This will also plant some
2849                 // kind of crashing instruction so that if by some fluke the bounds check didn't
2850                 // work, we'll crash in an easy-to-see way.
2851                 didAlreadyTerminate();
2852                 return;
2853             }
2854             base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2855         } else
2856             base = addressFor(virtualRegisterForArgument(1));
2857         
2858         LValue pointer = m_out.baseIndex(
2859             base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2860         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2861     }
2862     
2863     void compilePutByVal()
2864     {
2865         Edge child1 = m_graph.varArgChild(m_node, 0);
2866         Edge child2 = m_graph.varArgChild(m_node, 1);
2867         Edge child3 = m_graph.varArgChild(m_node, 2);
2868         Edge child4 = m_graph.varArgChild(m_node, 3);
2869         Edge child5 = m_graph.varArgChild(m_node, 4);
2870         
2871         switch (m_node->arrayMode().type()) {
2872         case Array::Generic: {
2873             V_JITOperation_EJJJ operation;
2874             if (m_node->op() == PutByValDirect) {
2875                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2876                     operation = operationPutByValDirectStrict;
2877                 else
2878                     operation = operationPutByValDirectNonStrict;
2879             } else {
2880                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2881                     operation = operationPutByValStrict;
2882                 else
2883                     operation = operationPutByValNonStrict;
2884             }
2885                 
2886             vmCall(
2887                 m_out.voidType, m_out.operation(operation), m_callFrame,
2888                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2889             return;
2890         }
2891             
2892         default:
2893             break;
2894         }
2895
2896         LValue base = lowCell(child1);
2897         LValue index = lowInt32(child2);
2898         LValue storage = lowStorage(child4);
2899         
2900         switch (m_node->arrayMode().type()) {
2901         case Array::Int32:
2902         case Array::Double:
2903         case Array::Contiguous: {
2904             LBasicBlock continuation = m_out.newBlock();
2905             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2906             
2907             switch (m_node->arrayMode().type()) {
2908             case Array::Int32:
2909             case Array::Contiguous: {
2910                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2911                 
2912                 if (m_node->arrayMode().type() == Array::Int32)
2913                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2914                 
2915                 TypedPointer elementPointer = m_out.baseIndex(
2916                     m_node->arrayMode().type() == Array::Int32 ?
2917                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2918                     storage, m_out.zeroExtPtr(index), provenValue(child2));
2919                 
2920                 if (m_node->op() == PutByValAlias) {
2921                     m_out.store64(value, elementPointer);
2922                     break;
2923                 }
2924                 
2925                 contiguousPutByValOutOfBounds(
2926                     codeBlock()->isStrictMode()
2927                     ? operationPutByValBeyondArrayBoundsStrict
2928                     : operationPutByValBeyondArrayBoundsNonStrict,
2929                     base, storage, index, value, continuation);
2930                 
2931                 m_out.store64(value, elementPointer);
2932                 break;
2933             }
2934                 
2935             case Array::Double: {
2936                 LValue value = lowDouble(child3);
2937                 
2938                 FTL_TYPE_CHECK(
2939                     doubleValue(value), child3, SpecDoubleReal,
2940                     m_out.doubleNotEqualOrUnordered(value, value));
2941                 
2942                 TypedPointer elementPointer = m_out.baseIndex(
2943                     m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2944                     provenValue(child2));
2945                 
2946                 if (m_node->op() == PutByValAlias) {
2947                     m_out.storeDouble(value, elementPointer);
2948                     break;
2949                 }
2950                 
2951                 contiguousPutByValOutOfBounds(
2952                     codeBlock()->isStrictMode()
2953                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2954                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2955                     base, storage, index, value, continuation);
2956                 
2957                 m_out.storeDouble(value, elementPointer);
2958                 break;
2959             }
2960                 
2961             default:
2962                 DFG_CRASH(m_graph, m_node, "Bad array type");
2963             }
2964
2965             m_out.jump(continuation);
2966             m_out.appendTo(continuation, outerLastNext);
2967             return;
2968         }
2969             
2970         default:
2971             TypedArrayType type = m_node->arrayMode().typedArrayType();
2972             
2973             if (isTypedView(type)) {
2974                 TypedPointer pointer = TypedPointer(
2975                     m_heaps.typedArrayProperties,
2976                     m_out.add(
2977                         storage,
2978                         m_out.shl(
2979                             m_out.zeroExt(index, m_out.intPtr),
2980                             m_out.constIntPtr(logElementSize(type)))));
2981                 
2982                 Output::StoreType storeType;
2983                 LValue valueToStore;
2984                 
2985                 if (isInt(type)) {
2986                     LValue intValue;
2987                     switch (child3.useKind()) {
2988                     case Int52RepUse:
2989                     case Int32Use: {
2990                         if (child3.useKind() == Int32Use)
2991                             intValue = lowInt32(child3);
2992                         else
2993                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2994
2995                         if (isClamped(type)) {
2996                             ASSERT(elementSize(type) == 1);
2997                             
2998                             LBasicBlock atLeastZero = m_out.newBlock();
2999                             LBasicBlock continuation = m_out.newBlock();
3000                             
3001                             Vector<ValueFromBlock, 2> intValues;
3002                             intValues.append(m_out.anchor(m_out.int32Zero));
3003                             m_out.branch(
3004                                 m_out.lessThan(intValue, m_out.int32Zero),
3005                                 unsure(continuation), unsure(atLeastZero));
3006                             
3007                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
3008                             
3009                             intValues.append(m_out.anchor(m_out.select(
3010                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
3011                                 m_out.constInt32(255),
3012                                 intValue)));
3013                             m_out.jump(continuation);
3014                             
3015                             m_out.appendTo(continuation, lastNext);
3016                             intValue = m_out.phi(m_out.int32, intValues);
3017                         }
3018                         break;
3019                     }
3020                         
3021                     case DoubleRepUse: {
3022                         LValue doubleValue = lowDouble(child3);
3023                         
3024                         if (isClamped(type)) {
3025                             ASSERT(elementSize(type) == 1);
3026                             
3027                             LBasicBlock atLeastZero = m_out.newBlock();
3028                             LBasicBlock withinRange = m_out.newBlock();
3029                             LBasicBlock continuation = m_out.newBlock();
3030                             
3031                             Vector<ValueFromBlock, 3> intValues;
3032                             intValues.append(m_out.anchor(m_out.int32Zero));
3033                             m_out.branch(
3034                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
3035                                 unsure(continuation), unsure(atLeastZero));
3036                             
3037                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
3038                             intValues.append(m_out.anchor(m_out.constInt32(255)));
3039                             m_out.branch(
3040                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
3041                                 unsure(continuation), unsure(withinRange));
3042                             
3043                             m_out.appendTo(withinRange, continuation);
3044                             intValues.append(m_out.anchor(m_out.doubleToInt(doubleValue)));
3045                             m_out.jump(continuation);
3046                             
3047                             m_out.appendTo(continuation, lastNext);
3048                             intValue = m_out.phi(m_out.int32, intValues);
3049                         } else
3050                             intValue = doubleToInt32(doubleValue);
3051                         break;
3052                     }
3053                         
3054                     default:
3055                         DFG_CRASH(m_graph, m_node, "Bad use kind");
3056                     }
3057
3058                     valueToStore = intValue;
3059                     switch (elementSize(type)) {
3060                     case 1:
3061                         storeType = Output::Store32As8;
3062                         break;
3063                     case 2:
3064                         storeType = Output::Store32As16;
3065                         break;
3066                     case 4:
3067                         storeType = Output::Store32;
3068                         break;
3069                     default:
3070                         DFG_CRASH(m_graph, m_node, "Bad element size");
3071                     }
3072                 } else /* !isInt(type) */ {
3073                     LValue value = lowDouble(child3);
3074                     switch (type) {
3075                     case TypeFloat32:
3076                         valueToStore = m_out.doubleToFloat(value);
3077                         storeType = Output::StoreFloat;
3078                         break;
3079                     case TypeFloat64:
3080                         valueToStore = value;
3081                         storeType = Output::StoreDouble;
3082                         break;
3083                     default:
3084                         DFG_CRASH(m_graph, m_node, "Bad typed array type");
3085                     }
3086                 }
3087                 
3088                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
3089                     m_out.store(valueToStore, pointer, storeType);
3090                 else {
3091                     LBasicBlock isInBounds = m_out.newBlock();
3092                     LBasicBlock continuation = m_out.newBlock();
3093                     
3094                     m_out.branch(
3095                         m_out.aboveOrEqual(index, lowInt32(child5)),
3096                         unsure(continuation), unsure(isInBounds));
3097                     
3098                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
3099                     m_out.store(valueToStore, pointer, storeType);
3100                     m_out.jump(continuation);
3101                     
3102                     m_out.appendTo(continuation, lastNext);
3103                 }
3104                 
3105                 return;
3106             }
3107             
3108             DFG_CRASH(m_graph, m_node, "Bad array type");
3109             break;
3110         }
3111     }
3112
3113     void compilePutAccessorById()
3114     {
3115         LValue base = lowCell(m_node->child1());
3116         LValue accessor = lowCell(m_node->child2());
3117         auto uid = m_graph.identifiers()[m_node->identifierNumber()];
3118         vmCall(
3119             m_out.voidType,
3120             m_out.operation(m_node->op() == PutGetterById ? operationPutGetterById : operationPutSetterById),
3121             m_callFrame, base, m_out.constIntPtr(uid), m_out.constInt32(m_node->accessorAttributes()), accessor);