Use constexpr instead of const in symbol definitions that are obviously constexpr.
[WebKit-https.git] / Source / JavaScriptCore / b3 / B3LowerToAir.cpp
1 /*
2  * Copyright (C) 2015-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "B3LowerToAir.h"
28
29 #if ENABLE(B3_JIT)
30
31 #include "AirBlockInsertionSet.h"
32 #include "AirCCallSpecial.h"
33 #include "AirCode.h"
34 #include "AirInsertionSet.h"
35 #include "AirInstInlines.h"
36 #include "AirPrintSpecial.h"
37 #include "AirStackSlot.h"
38 #include "B3ArgumentRegValue.h"
39 #include "B3AtomicValue.h"
40 #include "B3BasicBlockInlines.h"
41 #include "B3BlockWorklist.h"
42 #include "B3CCallValue.h"
43 #include "B3CheckSpecial.h"
44 #include "B3Commutativity.h"
45 #include "B3Dominators.h"
46 #include "B3ExtractValue.h"
47 #include "B3FenceValue.h"
48 #include "B3MemoryValueInlines.h"
49 #include "B3PatchpointSpecial.h"
50 #include "B3PatchpointValue.h"
51 #include "B3PhaseScope.h"
52 #include "B3PhiChildren.h"
53 #include "B3Procedure.h"
54 #include "B3SlotBaseValue.h"
55 #include "B3StackSlot.h"
56 #include "B3UpsilonValue.h"
57 #include "B3UseCounts.h"
58 #include "B3ValueInlines.h"
59 #include "B3Variable.h"
60 #include "B3VariableValue.h"
61 #include "B3WasmAddressValue.h"
62 #include <wtf/IndexMap.h>
63 #include <wtf/IndexSet.h>
64 #include <wtf/ListDump.h>
65
66 #if ASSERT_DISABLED
67 IGNORE_RETURN_TYPE_WARNINGS_BEGIN
68 #endif
69
70 namespace JSC { namespace B3 {
71
72 namespace {
73
74 namespace B3LowerToAirInternal {
75 static constexpr bool verbose = false;
76 }
77
78 using Arg = Air::Arg;
79 using Inst = Air::Inst;
80 using Code = Air::Code;
81 using Tmp = Air::Tmp;
82
83 // FIXME: We wouldn't need this if Air supported Width modifiers in Air::Kind.
84 // https://bugs.webkit.org/show_bug.cgi?id=169247
85 #define OPCODE_FOR_WIDTH(opcode, width) ( \
86     (width) == Width8 ? Air::opcode ## 8 : \
87     (width) == Width16 ? Air::opcode ## 16 :    \
88     (width) == Width32 ? Air::opcode ## 32 :    \
89     Air::opcode ## 64)
90 #define OPCODE_FOR_CANONICAL_WIDTH(opcode, width) ( \
91     (width) == Width64 ? Air::opcode ## 64 : Air::opcode ## 32)
92
93 class LowerToAir {
94 public:
95     LowerToAir(Procedure& procedure)
96         : m_valueToTmp(procedure.values().size())
97         , m_phiToTmp(procedure.values().size())
98         , m_blockToBlock(procedure.size())
99         , m_useCounts(procedure)
100         , m_phiChildren(procedure)
101         , m_dominators(procedure.dominators())
102         , m_procedure(procedure)
103         , m_code(procedure.code())
104         , m_blockInsertionSet(m_code)
105 #if CPU(X86) || CPU(X86_64)
106         , m_eax(X86Registers::eax)
107         , m_ecx(X86Registers::ecx)
108         , m_edx(X86Registers::edx)
109 #endif
110     {
111     }
112
113     void run()
114     {
115         using namespace Air;
116         for (B3::BasicBlock* block : m_procedure)
117             m_blockToBlock[block] = m_code.addBlock(block->frequency());
118
119         auto ensureTupleTmps = [&] (Value* tupleValue, auto& hashTable) {
120             hashTable.ensure(tupleValue, [&] {
121                 const auto tuple = m_procedure.tupleForType(tupleValue->type());
122                 Vector<Tmp> tmps(tuple.size());
123
124                 for (unsigned i = 0; i < tuple.size(); ++i)
125                     tmps[i] = tmpForType(tuple[i]);
126                 return tmps;
127             });
128         };
129
130         for (Value* value : m_procedure.values()) {
131             switch (value->opcode()) {
132             case Phi: {
133                 if (value->type().isTuple()) {
134                     ensureTupleTmps(value, m_tuplePhiToTmps);
135                     ensureTupleTmps(value, m_tupleValueToTmps);
136                     break;
137                 }
138
139                 m_phiToTmp[value] = m_code.newTmp(value->resultBank());
140                 if (B3LowerToAirInternal::verbose)
141                     dataLog("Phi tmp for ", *value, ": ", m_phiToTmp[value], "\n");
142                 break;
143             }
144             case Get:
145             case Patchpoint: {
146                 if (value->type().isTuple())
147                     ensureTupleTmps(value, m_tupleValueToTmps);
148                 break;
149             }
150             default:
151                 break;
152             }
153         }
154
155         for (B3::StackSlot* stack : m_procedure.stackSlots())
156             m_stackToStack.add(stack, m_code.addStackSlot(stack));
157         for (Variable* variable : m_procedure.variables()) {
158             auto addResult = m_variableToTmps.add(variable, Vector<Tmp, 1>(m_procedure.returnCount(variable->type())));
159             ASSERT(addResult.isNewEntry);
160             for (unsigned i = 0; i < m_procedure.returnCount(variable->type()); ++i)
161                 addResult.iterator->value[i] = tmpForType(variable->type().isNumeric() ? variable->type() : m_procedure.extractFromTuple(variable->type(), i));
162         }
163
164         // Figure out which blocks are not rare.
165         m_fastWorklist.push(m_procedure[0]);
166         while (B3::BasicBlock* block = m_fastWorklist.pop()) {
167             for (B3::FrequentedBlock& successor : block->successors()) {
168                 if (!successor.isRare())
169                     m_fastWorklist.push(successor.block());
170             }
171         }
172
173         m_procedure.resetValueOwners(); // Used by crossesInterference().
174
175         // Lower defs before uses on a global level. This is a good heuristic to lock down a
176         // hoisted address expression before we duplicate it back into the loop.
177         for (B3::BasicBlock* block : m_procedure.blocksInPreOrder()) {
178             m_block = block;
179
180             m_isRare = !m_fastWorklist.saw(block);
181
182             if (B3LowerToAirInternal::verbose)
183                 dataLog("Lowering Block ", *block, ":\n");
184             
185             // Make sure that the successors are set up correctly.
186             for (B3::FrequentedBlock successor : block->successors()) {
187                 m_blockToBlock[block]->successors().append(
188                     Air::FrequentedBlock(m_blockToBlock[successor.block()], successor.frequency()));
189             }
190
191             // Process blocks in reverse order so we see uses before defs. That's what allows us
192             // to match patterns effectively.
193             for (unsigned i = block->size(); i--;) {
194                 m_index = i;
195                 m_value = block->at(i);
196                 if (m_locked.contains(m_value))
197                     continue;
198                 m_insts.append(Vector<Inst>());
199                 if (B3LowerToAirInternal::verbose)
200                     dataLog("Lowering ", deepDump(m_procedure, m_value), ":\n");
201                 lower();
202                 if (B3LowerToAirInternal::verbose) {
203                     for (Inst& inst : m_insts.last())
204                         dataLog("    ", inst, "\n");
205                 }
206             }
207
208             finishAppendingInstructions(m_blockToBlock[block]);
209         }
210         
211         m_blockInsertionSet.execute();
212
213         Air::InsertionSet insertionSet(m_code);
214         for (Inst& inst : m_prologue)
215             insertionSet.insertInst(0, WTFMove(inst));
216         insertionSet.execute(m_code[0]);
217     }
218
219 private:
220     bool shouldCopyPropagate(Value* value)
221     {
222         switch (value->opcode()) {
223         case Trunc:
224         case Identity:
225         case Opaque:
226             return true;
227         default:
228             return false;
229         }
230     }
231
232     class ArgPromise {
233         WTF_MAKE_NONCOPYABLE(ArgPromise);
234     public:
235         ArgPromise() { }
236
237         ArgPromise(const Arg& arg, Value* valueToLock = nullptr)
238             : m_arg(arg)
239             , m_value(valueToLock)
240         {
241         }
242         
243         void swap(ArgPromise& other)
244         {
245             std::swap(m_arg, other.m_arg);
246             std::swap(m_value, other.m_value);
247             std::swap(m_wasConsumed, other.m_wasConsumed);
248             std::swap(m_wasWrapped, other.m_wasWrapped);
249             std::swap(m_traps, other.m_traps);
250         }
251         
252         ArgPromise(ArgPromise&& other)
253         {
254             swap(other);
255         }
256         
257         ArgPromise& operator=(ArgPromise&& other)
258         {
259             swap(other);
260             return *this;
261         }
262         
263         ~ArgPromise()
264         {
265             if (m_wasConsumed)
266                 RELEASE_ASSERT(m_wasWrapped);
267         }
268         
269         void setTraps(bool value)
270         {
271             m_traps = value;
272         }
273
274         static ArgPromise tmp(Value* value)
275         {
276             ArgPromise result;
277             result.m_value = value;
278             return result;
279         }
280
281         explicit operator bool() const { return m_arg || m_value; }
282
283         Arg::Kind kind() const
284         {
285             if (!m_arg && m_value)
286                 return Arg::Tmp;
287             return m_arg.kind();
288         }
289
290         const Arg& peek() const
291         {
292             return m_arg;
293         }
294
295         Arg consume(LowerToAir& lower)
296         {
297             m_wasConsumed = true;
298             if (!m_arg && m_value)
299                 return lower.tmp(m_value);
300             if (m_value)
301                 lower.commitInternal(m_value);
302             return m_arg;
303         }
304         
305         template<typename... Args>
306         Inst inst(Args&&... args)
307         {
308             Inst result(std::forward<Args>(args)...);
309             result.kind.effects |= m_traps;
310             m_wasWrapped = true;
311             return result;
312         }
313         
314     private:
315         // Three forms:
316         // Everything null: invalid.
317         // Arg non-null, value null: just use the arg, nothing special.
318         // Arg null, value non-null: it's a tmp, pin it when necessary.
319         // Arg non-null, value non-null: use the arg, lock the value.
320         Arg m_arg;
321         Value* m_value { nullptr };
322         bool m_wasConsumed { false };
323         bool m_wasWrapped { false };
324         bool m_traps { false };
325     };
326
327     // Consider using tmpPromise() in cases where you aren't sure that you want to pin the value yet.
328     // Here are three canonical ways of using tmp() and tmpPromise():
329     //
330     // Idiom #1: You know that you want a tmp() and you know that it will be valid for the
331     // instruction you're emitting.
332     //
333     //     append(Foo, tmp(bar));
334     //
335     // Idiom #2: You don't know if you want to use a tmp() because you haven't determined if the
336     // instruction will accept it, so you query first. Note that the call to tmp() happens only after
337     // you are sure that you will use it.
338     //
339     //     if (isValidForm(Foo, Arg::Tmp))
340     //         append(Foo, tmp(bar))
341     //
342     // Idiom #3: Same as Idiom #2, but using tmpPromise. Notice that this calls consume() only after
343     // it's sure it will use the tmp. That's deliberate. Also note that you're required to pass any
344     // Inst you create with consumed promises through that promise's inst() function.
345     //
346     //     ArgPromise promise = tmpPromise(bar);
347     //     if (isValidForm(Foo, promise.kind()))
348     //         append(promise.inst(Foo, promise.consume(*this)))
349     //
350     // In both idiom #2 and idiom #3, we don't pin the value to a temporary except when we actually
351     // emit the instruction. Both tmp() and tmpPromise().consume(*this) will pin it. Pinning means
352     // that we will henceforth require that the value of 'bar' is generated as a separate
353     // instruction. We don't want to pin the value to a temporary if we might change our minds, and
354     // pass an address operand representing 'bar' to Foo instead.
355     //
356     // Because tmp() pins, the following is not an idiom you should use:
357     //
358     //     Tmp tmp = this->tmp(bar);
359     //     if (isValidForm(Foo, tmp.kind()))
360     //         append(Foo, tmp);
361     //
362     // That's because if isValidForm() returns false, you will have already pinned the 'bar' to a
363     // temporary. You might later want to try to do something like loadPromise(), and that will fail.
364     // This arises in operations that have both a Addr,Tmp and Tmp,Addr forms. The following code
365     // seems right, but will actually fail to ever match the Tmp,Addr form because by then, the right
366     // value is already pinned.
367     //
368     //     auto tryThings = [this] (const Arg& left, const Arg& right) {
369     //         if (isValidForm(Foo, left.kind(), right.kind()))
370     //             return Inst(Foo, m_value, left, right);
371     //         return Inst();
372     //     };
373     //     if (Inst result = tryThings(loadAddr(left), tmp(right)))
374     //         return result;
375     //     if (Inst result = tryThings(tmp(left), loadAddr(right))) // this never succeeds.
376     //         return result;
377     //     return Inst(Foo, m_value, tmp(left), tmp(right));
378     //
379     // If you imagine that loadAddr(value) is just loadPromise(value).consume(*this), then this code
380     // will run correctly - it will generate OK code - but the second form is never matched.
381     // loadAddr(right) will never succeed because it will observe that 'right' is already pinned.
382     // Of course, it's exactly because of the risky nature of such code that we don't have a
383     // loadAddr() helper and require you to balance ArgPromise's in code like this. Such code will
384     // work fine if written as:
385     //
386     //     auto tryThings = [this] (ArgPromise& left, ArgPromise& right) {
387     //         if (isValidForm(Foo, left.kind(), right.kind()))
388     //             return left.inst(right.inst(Foo, m_value, left.consume(*this), right.consume(*this)));
389     //         return Inst();
390     //     };
391     //     if (Inst result = tryThings(loadPromise(left), tmpPromise(right)))
392     //         return result;
393     //     if (Inst result = tryThings(tmpPromise(left), loadPromise(right)))
394     //         return result;
395     //     return Inst(Foo, m_value, tmp(left), tmp(right));
396     //
397     // Notice that we did use tmp in the fall-back case at the end, because by then, we know for sure
398     // that we want a tmp. But using tmpPromise in the tryThings() calls ensures that doing so
399     // doesn't prevent us from trying loadPromise on the same value.
400     Tmp tmp(Value* value)
401     {
402         Tmp& tmp = m_valueToTmp[value];
403         if (!tmp) {
404             while (shouldCopyPropagate(value))
405                 value = value->child(0);
406
407             if (value->opcode() == FramePointer)
408                 return Tmp(GPRInfo::callFrameRegister);
409
410             Tmp& realTmp = m_valueToTmp[value];
411             if (!realTmp) {
412                 realTmp = m_code.newTmp(value->resultBank());
413                 if (m_procedure.isFastConstant(value->key()))
414                     m_code.addFastTmp(realTmp);
415                 if (B3LowerToAirInternal::verbose)
416                     dataLog("Tmp for ", *value, ": ", realTmp, "\n");
417             }
418             tmp = realTmp;
419         }
420         return tmp;
421     }
422
423     ArgPromise tmpPromise(Value* value)
424     {
425         return ArgPromise::tmp(value);
426     }
427
428     Tmp tmpForType(Type type)
429     {
430         return m_code.newTmp(bankForType(type));
431     }
432
433     const Vector<Tmp>& tmpsForTuple(Value* tupleValue)
434     {
435         ASSERT(tupleValue->type().isTuple());
436
437         switch (tupleValue->opcode()) {
438         case Phi:
439         case Patchpoint: {
440             return m_tupleValueToTmps.find(tupleValue)->value;
441         }
442         case Get:
443         case Set:
444             return m_variableToTmps.find(tupleValue->as<VariableValue>()->variable())->value;
445         default:
446             break;
447         }
448         RELEASE_ASSERT_NOT_REACHED();
449     }
450
451     bool canBeInternal(Value* value)
452     {
453         // If one of the internal things has already been computed, then we don't want to cause
454         // it to be recomputed again.
455         if (m_valueToTmp[value])
456             return false;
457         
458         // We require internals to have only one use - us. It's not clear if this should be numUses() or
459         // numUsingInstructions(). Ideally, it would be numUsingInstructions(), except that it's not clear
460         // if we'd actually do the right thing when matching over such a DAG pattern. For now, it simply
461         // doesn't matter because we don't implement patterns that would trigger this.
462         if (m_useCounts.numUses(value) != 1)
463             return false;
464
465         return true;
466     }
467
468     // If you ask canBeInternal() and then construct something from that, and you commit to emitting
469     // that code, then you must commitInternal() on that value. This is tricky, and you only need to
470     // do it if you're pattern matching by hand rather than using the patterns language. Long story
471     // short, you should avoid this by using the pattern matcher to match patterns.
472     void commitInternal(Value* value)
473     {
474         if (value)
475             m_locked.add(value);
476     }
477
478     bool crossesInterference(Value* value)
479     {
480         // If it's in a foreign block, then be conservative. We could handle this if we were
481         // willing to do heavier analysis. For example, if we had liveness, then we could label
482         // values as "crossing interference" if they interfere with anything that they are live
483         // across. But, it's not clear how useful this would be.
484         if (value->owner != m_value->owner)
485             return true;
486
487         Effects effects = value->effects();
488
489         for (unsigned i = m_index; i--;) {
490             Value* otherValue = m_block->at(i);
491             if (otherValue == value)
492                 return false;
493             if (effects.interferes(otherValue->effects()))
494                 return true;
495         }
496
497         ASSERT_NOT_REACHED();
498         return true;
499     }
500
501     template<typename Int, typename = Value::IsLegalOffset<Int>>
502     Optional<unsigned> scaleForShl(Value* shl, Int offset, Optional<Width> width = WTF::nullopt)
503     {
504         if (shl->opcode() != Shl)
505             return WTF::nullopt;
506         if (!shl->child(1)->hasInt32())
507             return WTF::nullopt;
508         unsigned logScale = shl->child(1)->asInt32();
509         if (shl->type() == Int32)
510             logScale &= 31;
511         else
512             logScale &= 63;
513         // Use 64-bit math to perform the shift so that <<32 does the right thing, but then switch
514         // to signed since that's what all of our APIs want.
515         int64_t bigScale = static_cast<uint64_t>(1) << static_cast<uint64_t>(logScale);
516         if (!isRepresentableAs<int32_t>(bigScale))
517             return WTF::nullopt;
518         unsigned scale = static_cast<int32_t>(bigScale);
519         if (!Arg::isValidIndexForm(scale, offset, width))
520             return WTF::nullopt;
521         return scale;
522     }
523     
524     // This turns the given operand into an address.
525     template<typename Int, typename = Value::IsLegalOffset<Int>>
526     Arg effectiveAddr(Value* address, Int offset, Width width)
527     {
528         ASSERT(Arg::isValidAddrForm(offset, width));
529         
530         auto fallback = [&] () -> Arg {
531             return Arg::addr(tmp(address), offset);
532         };
533         
534         static constexpr unsigned lotsOfUses = 10; // This is arbitrary and we should tune it eventually.
535
536         // Only match if the address value isn't used in some large number of places.
537         if (m_useCounts.numUses(address) > lotsOfUses)
538             return fallback();
539         
540         switch (address->opcode()) {
541         case Add: {
542             Value* left = address->child(0);
543             Value* right = address->child(1);
544
545             auto tryIndex = [&] (Value* index, Value* base) -> Arg {
546                 Optional<unsigned> scale = scaleForShl(index, offset, width);
547                 if (!scale)
548                     return Arg();
549                 if (m_locked.contains(index->child(0)) || m_locked.contains(base))
550                     return Arg();
551                 return Arg::index(tmp(base), tmp(index->child(0)), *scale, offset);
552             };
553
554             if (Arg result = tryIndex(left, right))
555                 return result;
556             if (Arg result = tryIndex(right, left))
557                 return result;
558
559             if (m_locked.contains(left) || m_locked.contains(right)
560                 || !Arg::isValidIndexForm(1, offset, width))
561                 return fallback();
562             
563             return Arg::index(tmp(left), tmp(right), 1, offset);
564         }
565
566         case Shl: {
567             Value* left = address->child(0);
568
569             // We'll never see child(1)->isInt32(0), since that would have been reduced. If the shift
570             // amount is greater than 1, then there isn't really anything smart that we could do here.
571             // We avoid using baseless indexes because their encoding isn't particularly efficient.
572             if (m_locked.contains(left) || !address->child(1)->isInt32(1)
573                 || !Arg::isValidIndexForm(1, offset, width))
574                 return fallback();
575
576             return Arg::index(tmp(left), tmp(left), 1, offset);
577         }
578
579         case FramePointer:
580             return Arg::addr(Tmp(GPRInfo::callFrameRegister), offset);
581
582         case SlotBase:
583             return Arg::stack(m_stackToStack.get(address->as<SlotBaseValue>()->slot()), offset);
584
585         case WasmAddress: {
586             WasmAddressValue* wasmAddress = address->as<WasmAddressValue>();
587             Value* pointer = wasmAddress->child(0);
588             if (!Arg::isValidIndexForm(1, offset, width) || m_locked.contains(pointer))
589                 return fallback();
590
591             // FIXME: We should support ARM64 LDR 32-bit addressing, which will
592             // allow us to fuse a Shl ptr, 2 into the address. Additionally, and
593             // perhaps more importantly, it would allow us to avoid a truncating
594             // move. See: https://bugs.webkit.org/show_bug.cgi?id=163465
595
596             return Arg::index(Tmp(wasmAddress->pinnedGPR()), tmp(pointer), 1, offset);
597         }
598
599         default:
600             return fallback();
601         }
602     }
603
604     // This gives you the address of the given Load or Store. If it's not a Load or Store, then
605     // it returns Arg().
606     Arg addr(Value* memoryValue)
607     {
608         MemoryValue* value = memoryValue->as<MemoryValue>();
609         if (!value)
610             return Arg();
611         
612         if (value->requiresSimpleAddr())
613             return Arg::simpleAddr(tmp(value->lastChild()));
614
615         Value::OffsetType offset = value->offset();
616         Width width = value->accessWidth();
617
618         Arg result = effectiveAddr(value->lastChild(), offset, width);
619         RELEASE_ASSERT(result.isValidForm(width));
620
621         return result;
622     }
623     
624     template<typename... Args>
625     Inst trappingInst(bool traps, Args&&... args)
626     {
627         Inst result(std::forward<Args>(args)...);
628         result.kind.effects |= traps;
629         return result;
630     }
631     
632     template<typename... Args>
633     Inst trappingInst(Value* value, Args&&... args)
634     {
635         return trappingInst(value->traps(), std::forward<Args>(args)...);
636     }
637     
638     ArgPromise loadPromiseAnyOpcode(Value* loadValue)
639     {
640         RELEASE_ASSERT(loadValue->as<MemoryValue>());
641         if (!canBeInternal(loadValue))
642             return Arg();
643         if (crossesInterference(loadValue))
644             return Arg();
645         // On x86, all loads have fences. Doing this kind of instruction selection will move the load,
646         // but that's fine because our interference analysis stops the motion of fences around other
647         // fences. So, any load motion we introduce here would not be observable.
648         if (!isX86() && loadValue->as<MemoryValue>()->hasFence())
649             return Arg();
650         Arg loadAddr = addr(loadValue);
651         RELEASE_ASSERT(loadAddr);
652         ArgPromise result(loadAddr, loadValue);
653         if (loadValue->traps())
654             result.setTraps(true);
655         return result;
656     }
657
658     ArgPromise loadPromise(Value* loadValue, B3::Opcode loadOpcode)
659     {
660         if (loadValue->opcode() != loadOpcode)
661             return Arg();
662         return loadPromiseAnyOpcode(loadValue);
663     }
664
665     ArgPromise loadPromise(Value* loadValue)
666     {
667         return loadPromise(loadValue, Load);
668     }
669
670     Arg imm(int64_t intValue)
671     {
672         if (Arg::isValidImmForm(intValue))
673             return Arg::imm(intValue);
674         return Arg();
675     }
676
677     Arg imm(Value* value)
678     {
679         if (value->hasInt())
680             return imm(value->asInt());
681         return Arg();
682     }
683
684     Arg bitImm(Value* value)
685     {
686         if (value->hasInt()) {
687             int64_t intValue = value->asInt();
688             if (Arg::isValidBitImmForm(intValue))
689                 return Arg::bitImm(intValue);
690         }
691         return Arg();
692     }
693
694     Arg bitImm64(Value* value)
695     {
696         if (value->hasInt()) {
697             int64_t intValue = value->asInt();
698             if (Arg::isValidBitImm64Form(intValue))
699                 return Arg::bitImm64(intValue);
700         }
701         return Arg();
702     }
703
704     Arg immOrTmp(Value* value)
705     {
706         if (Arg result = imm(value))
707             return result;
708         return tmp(value);
709     }
710
711     template<typename Functor>
712     void forEachImmOrTmp(Value* value, const Functor& func)
713     {
714         ASSERT(value->type() != Void);
715         if (!value->type().isTuple()) {
716             func(immOrTmp(value), value->type(), 0);
717             return;
718         }
719
720         const Vector<Type>& tuple = m_procedure.tupleForType(value->type());
721         const auto& tmps = tmpsForTuple(value);
722         for (unsigned i = 0; i < tuple.size(); ++i)
723             func(tmps[i], tuple[i], i);
724     }
725
726     // By convention, we use Oops to mean "I don't know".
727     Air::Opcode tryOpcodeForType(
728         Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Type type)
729     {
730         Air::Opcode opcode;
731         switch (type.kind()) {
732         case Int32:
733             opcode = opcode32;
734             break;
735         case Int64:
736             opcode = opcode64;
737             break;
738         case Float:
739             opcode = opcodeFloat;
740             break;
741         case Double:
742             opcode = opcodeDouble;
743             break;
744         default:
745             opcode = Air::Oops;
746             break;
747         }
748
749         return opcode;
750     }
751
752     Air::Opcode tryOpcodeForType(Air::Opcode opcode32, Air::Opcode opcode64, Type type)
753     {
754         return tryOpcodeForType(opcode32, opcode64, Air::Oops, Air::Oops, type);
755     }
756
757     Air::Opcode opcodeForType(
758         Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Type type)
759     {
760         Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, type);
761         RELEASE_ASSERT(opcode != Air::Oops);
762         return opcode;
763     }
764
765     Air::Opcode opcodeForType(Air::Opcode opcode32, Air::Opcode opcode64, Type type)
766     {
767         return tryOpcodeForType(opcode32, opcode64, Air::Oops, Air::Oops, type);
768     }
769
770     template<Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble = Air::Oops, Air::Opcode opcodeFloat = Air::Oops>
771     void appendUnOp(Value* value)
772     {
773         Air::Opcode opcode = opcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, value->type());
774         
775         Tmp result = tmp(m_value);
776
777         // Two operand forms like:
778         //     Op a, b
779         // mean something like:
780         //     b = Op a
781
782         ArgPromise addr = loadPromise(value);
783         if (isValidForm(opcode, addr.kind(), Arg::Tmp)) {
784             append(addr.inst(opcode, m_value, addr.consume(*this), result));
785             return;
786         }
787
788         if (isValidForm(opcode, Arg::Tmp, Arg::Tmp)) {
789             append(opcode, tmp(value), result);
790             return;
791         }
792
793         ASSERT(value->type() == m_value->type());
794         append(relaxedMoveForType(m_value->type()), tmp(value), result);
795         append(opcode, result);
796     }
797
798     // Call this method when doing two-operand lowering of a commutative operation. You have a choice of
799     // which incoming Value is moved into the result. This will select which one is likely to be most
800     // profitable to use as the result. Doing the right thing can have big performance consequences in tight
801     // kernels.
802     bool preferRightForResult(Value* left, Value* right)
803     {
804         // The default is to move left into result, because that's required for non-commutative instructions.
805         // The value that we want to move into result position is the one that dies here. So, if we're
806         // compiling a commutative operation and we know that actually right is the one that dies right here,
807         // then we can flip things around to help coalescing, which then kills the move instruction.
808         //
809         // But it's more complicated:
810         // - Used-once is a bad estimate of whether the variable dies here.
811         // - A child might be a candidate for coalescing with this value.
812         //
813         // Currently, we have machinery in place to recognize super obvious forms of the latter issue.
814         
815         // We recognize when a child is a Phi that has this value as one of its children. We're very
816         // conservative about this; for example we don't even consider transitive Phi children.
817         bool leftIsPhiWithThis = m_phiChildren[left].transitivelyUses(m_value);
818         bool rightIsPhiWithThis = m_phiChildren[right].transitivelyUses(m_value);
819
820         if (leftIsPhiWithThis != rightIsPhiWithThis)
821             return rightIsPhiWithThis;
822
823         if (m_useCounts.numUsingInstructions(right) != 1)
824             return false;
825         
826         if (m_useCounts.numUsingInstructions(left) != 1)
827             return true;
828
829         // The use count might be 1 if the variable is live around a loop. We can guarantee that we
830         // pick the variable that is least likely to suffer this problem if we pick the one that
831         // is closest to us in an idom walk. By convention, we slightly bias this in favor of
832         // returning true.
833
834         // We cannot prefer right if right is further away in an idom walk.
835         if (m_dominators.strictlyDominates(right->owner, left->owner))
836             return false;
837
838         return true;
839     }
840     
841     template<Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Commutativity commutativity = NotCommutative>
842     void appendBinOp(Value* left, Value* right)
843     {
844         Air::Opcode opcode = opcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, left->type());
845         
846         Tmp result = tmp(m_value);
847         
848         // Three-operand forms like:
849         //     Op a, b, c
850         // mean something like:
851         //     c = a Op b
852
853         if (isValidForm(opcode, Arg::Imm, Arg::Tmp, Arg::Tmp)) {
854             if (commutativity == Commutative) {
855                 if (imm(right)) {
856                     append(opcode, imm(right), tmp(left), result);
857                     return;
858                 }
859             } else {
860                 // A non-commutative operation could have an immediate in left.
861                 if (imm(left)) {
862                     append(opcode, imm(left), tmp(right), result);
863                     return;
864                 }
865             }
866         }
867
868         if (isValidForm(opcode, Arg::BitImm, Arg::Tmp, Arg::Tmp)) {
869             if (commutativity == Commutative) {
870                 if (Arg rightArg = bitImm(right)) {
871                     append(opcode, rightArg, tmp(left), result);
872                     return;
873                 }
874             } else {
875                 // A non-commutative operation could have an immediate in left.
876                 if (Arg leftArg = bitImm(left)) {
877                     append(opcode, leftArg, tmp(right), result);
878                     return;
879                 }
880             }
881         }
882
883         if (isValidForm(opcode, Arg::BitImm64, Arg::Tmp, Arg::Tmp)) {
884             if (commutativity == Commutative) {
885                 if (Arg rightArg = bitImm64(right)) {
886                     append(opcode, rightArg, tmp(left), result);
887                     return;
888                 }
889             } else {
890                 // A non-commutative operation could have an immediate in left.
891                 if (Arg leftArg = bitImm64(left)) {
892                     append(opcode, leftArg, tmp(right), result);
893                     return;
894                 }
895             }
896         }
897
898         if (imm(right) && isValidForm(opcode, Arg::Tmp, Arg::Imm, Arg::Tmp)) {
899             append(opcode, tmp(left), imm(right), result);
900             return;
901         }
902
903         // Note that no extant architecture has a three-operand form of binary operations that also
904         // load from memory. If such an abomination did exist, we would handle it somewhere around
905         // here.
906
907         // Two-operand forms like:
908         //     Op a, b
909         // mean something like:
910         //     b = b Op a
911
912         // At this point, we prefer versions of the operation that have a fused load or an immediate
913         // over three operand forms.
914
915         if (left != right) {
916             ArgPromise leftAddr = loadPromise(left);
917             if (isValidForm(opcode, leftAddr.kind(), Arg::Tmp, Arg::Tmp)) {
918                 append(leftAddr.inst(opcode, m_value, leftAddr.consume(*this), tmp(right), result));
919                 return;
920             }
921
922             if (commutativity == Commutative) {
923                 if (isValidForm(opcode, leftAddr.kind(), Arg::Tmp)) {
924                     append(relaxedMoveForType(m_value->type()), tmp(right), result);
925                     append(leftAddr.inst(opcode, m_value, leftAddr.consume(*this), result));
926                     return;
927                 }
928             }
929
930             ArgPromise rightAddr = loadPromise(right);
931             if (isValidForm(opcode, Arg::Tmp, rightAddr.kind(), Arg::Tmp)) {
932                 append(rightAddr.inst(opcode, m_value, tmp(left), rightAddr.consume(*this), result));
933                 return;
934             }
935
936             if (commutativity == Commutative) {
937                 if (isValidForm(opcode, rightAddr.kind(), Arg::Tmp, Arg::Tmp)) {
938                     append(rightAddr.inst(opcode, m_value, rightAddr.consume(*this), tmp(left), result));
939                     return;
940                 }
941             }
942
943             if (isValidForm(opcode, rightAddr.kind(), Arg::Tmp)) {
944                 append(relaxedMoveForType(m_value->type()), tmp(left), result);
945                 append(rightAddr.inst(opcode, m_value, rightAddr.consume(*this), result));
946                 return;
947             }
948         }
949
950         if (imm(right) && isValidForm(opcode, Arg::Imm, Arg::Tmp)) {
951             append(relaxedMoveForType(m_value->type()), tmp(left), result);
952             append(opcode, imm(right), result);
953             return;
954         }
955
956         if (isValidForm(opcode, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
957             append(opcode, tmp(left), tmp(right), result);
958             return;
959         }
960
961         if (commutativity == Commutative && preferRightForResult(left, right)) {
962             append(relaxedMoveForType(m_value->type()), tmp(right), result);
963             append(opcode, tmp(left), result);
964             return;
965         }
966         
967         append(relaxedMoveForType(m_value->type()), tmp(left), result);
968         append(opcode, tmp(right), result);
969     }
970
971     template<Air::Opcode opcode32, Air::Opcode opcode64, Commutativity commutativity = NotCommutative>
972     void appendBinOp(Value* left, Value* right)
973     {
974         appendBinOp<opcode32, opcode64, Air::Oops, Air::Oops, commutativity>(left, right);
975     }
976
977     template<Air::Opcode opcode32, Air::Opcode opcode64>
978     void appendShift(Value* value, Value* amount)
979     {
980         using namespace Air;
981         Air::Opcode opcode = opcodeForType(opcode32, opcode64, value->type());
982         
983         if (imm(amount)) {
984             if (isValidForm(opcode, Arg::Tmp, Arg::Imm, Arg::Tmp)) {
985                 append(opcode, tmp(value), imm(amount), tmp(m_value));
986                 return;
987             }
988             if (isValidForm(opcode, Arg::Imm, Arg::Tmp)) {
989                 append(Move, tmp(value), tmp(m_value));
990                 append(opcode, imm(amount), tmp(m_value));
991                 return;
992             }
993         }
994
995         if (isValidForm(opcode, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
996             append(opcode, tmp(value), tmp(amount), tmp(m_value));
997             return;
998         }
999
1000         append(Move, tmp(value), tmp(m_value));
1001         append(Move, tmp(amount), m_ecx);
1002         append(opcode, m_ecx, tmp(m_value));
1003     }
1004
1005     template<Air::Opcode opcode32, Air::Opcode opcode64>
1006     bool tryAppendStoreUnOp(Value* value)
1007     {
1008         Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, value->type());
1009         if (opcode == Air::Oops)
1010             return false;
1011         
1012         Arg storeAddr = addr(m_value);
1013         ASSERT(storeAddr);
1014
1015         ArgPromise loadPromise = this->loadPromise(value);
1016         if (loadPromise.peek() != storeAddr)
1017             return false;
1018
1019         if (!isValidForm(opcode, storeAddr.kind()))
1020             return false;
1021         
1022         loadPromise.consume(*this);
1023         append(trappingInst(m_value, loadPromise.inst(opcode, m_value, storeAddr)));
1024         return true;
1025     }
1026
1027     template<
1028         Air::Opcode opcode32, Air::Opcode opcode64, Commutativity commutativity = NotCommutative>
1029     bool tryAppendStoreBinOp(Value* left, Value* right)
1030     {
1031         RELEASE_ASSERT(m_value->as<MemoryValue>());
1032         
1033         Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, left->type());
1034         if (opcode == Air::Oops)
1035             return false;
1036         
1037         if (m_value->as<MemoryValue>()->hasFence())
1038             return false;
1039         
1040         Arg storeAddr = addr(m_value);
1041         ASSERT(storeAddr);
1042
1043         auto getLoadPromise = [&] (Value* load) -> ArgPromise {
1044             switch (m_value->opcode()) {
1045             case B3::Store:
1046                 if (load->opcode() != B3::Load)
1047                     return ArgPromise();
1048                 break;
1049             case B3::Store8:
1050                 if (load->opcode() != B3::Load8Z && load->opcode() != B3::Load8S)
1051                     return ArgPromise();
1052                 break;
1053             case B3::Store16:
1054                 if (load->opcode() != B3::Load16Z && load->opcode() != B3::Load16S)
1055                     return ArgPromise();
1056                 break;
1057             default:
1058                 return ArgPromise();
1059             }
1060             return loadPromiseAnyOpcode(load);
1061         };
1062         
1063         ArgPromise loadPromise;
1064         Value* otherValue = nullptr;
1065
1066         loadPromise = getLoadPromise(left);
1067         if (loadPromise.peek() == storeAddr)
1068             otherValue = right;
1069         else if (commutativity == Commutative) {
1070             loadPromise = getLoadPromise(right);
1071             if (loadPromise.peek() == storeAddr)
1072                 otherValue = left;
1073         }
1074
1075         if (!otherValue)
1076             return false;
1077
1078         if (isValidForm(opcode, Arg::Imm, storeAddr.kind()) && imm(otherValue)) {
1079             loadPromise.consume(*this);
1080             append(trappingInst(m_value, loadPromise.inst(opcode, m_value, imm(otherValue), storeAddr)));
1081             return true;
1082         }
1083
1084         if (!isValidForm(opcode, Arg::Tmp, storeAddr.kind()))
1085             return false;
1086
1087         loadPromise.consume(*this);
1088         append(trappingInst(m_value, loadPromise.inst(opcode, m_value, tmp(otherValue), storeAddr)));
1089         return true;
1090     }
1091
1092     Inst createStore(Air::Kind move, Value* value, const Arg& dest)
1093     {
1094         using namespace Air;
1095         if (auto imm_value = imm(value)) {
1096             if (isARM64() && imm_value.value() == 0) {
1097                 switch (move.opcode) {
1098                 default:
1099                     break;
1100                 case Air::Move32:
1101                     if (isValidForm(StoreZero32, dest.kind()) && dest.isValidForm(Width32))
1102                         return Inst(StoreZero32, m_value, dest);
1103                     break;
1104                 case Air::Move:
1105                     if (isValidForm(StoreZero64, dest.kind()) && dest.isValidForm(Width64))
1106                         return Inst(StoreZero64, m_value, dest);
1107                     break;
1108                 }
1109             }
1110             if (isValidForm(move.opcode, Arg::Imm, dest.kind()))
1111                 return Inst(move, m_value, imm_value, dest);
1112         }
1113
1114         return Inst(move, m_value, tmp(value), dest);
1115     }
1116     
1117     Air::Opcode storeOpcode(Width width, Bank bank)
1118     {
1119         using namespace Air;
1120         switch (width) {
1121         case Width8:
1122             RELEASE_ASSERT(bank == GP);
1123             return Air::Store8;
1124         case Width16:
1125             RELEASE_ASSERT(bank == GP);
1126             return Air::Store16;
1127         case Width32:
1128             switch (bank) {
1129             case GP:
1130                 return Move32;
1131             case FP:
1132                 return MoveFloat;
1133             }
1134             break;
1135         case Width64:
1136             RELEASE_ASSERT(is64Bit());
1137             switch (bank) {
1138             case GP:
1139                 return Move;
1140             case FP:
1141                 return MoveDouble;
1142             }
1143             break;
1144         }
1145         RELEASE_ASSERT_NOT_REACHED();
1146     }
1147     
1148     void appendStore(Value* value, const Arg& dest)
1149     {
1150         using namespace Air;
1151         MemoryValue* memory = value->as<MemoryValue>();
1152         RELEASE_ASSERT(memory->isStore());
1153
1154         Air::Kind kind;
1155         if (memory->hasFence()) {
1156             RELEASE_ASSERT(memory->accessBank() == GP);
1157             
1158             if (isX86()) {
1159                 kind = OPCODE_FOR_WIDTH(Xchg, memory->accessWidth());
1160                 kind.effects = true;
1161                 Tmp swapTmp = m_code.newTmp(GP);
1162                 append(relaxedMoveForType(memory->accessType()), tmp(memory->child(0)), swapTmp);
1163                 append(kind, swapTmp, dest);
1164                 return;
1165             }
1166             
1167             kind = OPCODE_FOR_WIDTH(StoreRel, memory->accessWidth());
1168         } else
1169             kind = storeOpcode(memory->accessWidth(), memory->accessBank());
1170         
1171         kind.effects |= memory->traps();
1172         
1173         append(createStore(kind, memory->child(0), dest));
1174     }
1175
1176     Air::Opcode moveForType(Type type)
1177     {
1178         using namespace Air;
1179         switch (type.kind()) {
1180         case Int32:
1181             return Move32;
1182         case Int64:
1183             RELEASE_ASSERT(is64Bit());
1184             return Move;
1185         case Float:
1186             return MoveFloat;
1187         case Double:
1188             return MoveDouble;
1189         case Void:
1190         case Tuple:
1191             break;
1192         }
1193         RELEASE_ASSERT_NOT_REACHED();
1194         return Air::Oops;
1195     }
1196
1197     Air::Opcode relaxedMoveForType(Type type)
1198     {
1199         using namespace Air;
1200         switch (type.kind()) {
1201         case Int32:
1202         case Int64:
1203             // For Int32, we could return Move or Move32. It's a trade-off.
1204             //
1205             // Move32: Using Move32 guarantees that we use the narrower move, but in cases where the
1206             //     register allocator can't prove that the variables involved are 32-bit, this will
1207             //     disable coalescing.
1208             //
1209             // Move: Using Move guarantees that the register allocator can coalesce normally, but in
1210             //     cases where it can't prove that the variables are 32-bit and it doesn't coalesce,
1211             //     this will force us to use a full 64-bit Move instead of the slightly cheaper
1212             //     32-bit Move32.
1213             //
1214             // Coalescing is a lot more profitable than turning Move into Move32. So, it's better to
1215             // use Move here because in cases where the register allocator cannot prove that
1216             // everything is 32-bit, we still get coalescing.
1217             return Move;
1218         case Float:
1219             // MoveFloat is always coalescable and we never convert MoveDouble to MoveFloat, so we
1220             // should use MoveFloat when we know that the temporaries involved are 32-bit.
1221             return MoveFloat;
1222         case Double:
1223             return MoveDouble;
1224         case Void:
1225         case Tuple:
1226             break;
1227         }
1228         RELEASE_ASSERT_NOT_REACHED();
1229         return Air::Oops;
1230     }
1231
1232 #if ENABLE(MASM_PROBE)
1233     template<typename... Arguments>
1234     void print(Arguments&&... arguments)
1235     {
1236         Value* origin = m_value;
1237         print(origin, std::forward<Arguments>(arguments)...);
1238     }
1239
1240     template<typename... Arguments>
1241     void print(Value* origin, Arguments&&... arguments)
1242     {
1243         auto printList = Printer::makePrintRecordList(arguments...);
1244         auto printSpecial = static_cast<Air::PrintSpecial*>(m_code.addSpecial(makeUnique<Air::PrintSpecial>(printList)));
1245         Inst inst(Air::Patch, origin, Arg::special(printSpecial));
1246         Printer::appendAirArgs(inst, std::forward<Arguments>(arguments)...);
1247         append(WTFMove(inst));
1248     }
1249 #endif // ENABLE(MASM_PROBE)
1250
1251     template<typename... Arguments>
1252     void append(Air::Kind kind, Arguments&&... arguments)
1253     {
1254         m_insts.last().append(Inst(kind, m_value, std::forward<Arguments>(arguments)...));
1255     }
1256     
1257     template<typename... Arguments>
1258     void appendTrapping(Air::Kind kind, Arguments&&... arguments)
1259     {
1260         m_insts.last().append(trappingInst(m_value, kind, m_value, std::forward<Arguments>(arguments)...));
1261     }
1262     
1263     void append(Inst&& inst)
1264     {
1265         m_insts.last().append(WTFMove(inst));
1266     }
1267     void append(const Inst& inst)
1268     {
1269         m_insts.last().append(inst);
1270     }
1271     
1272     void finishAppendingInstructions(Air::BasicBlock* target)
1273     {
1274         // Now append the instructions. m_insts contains them in reverse order, so we process
1275         // it in reverse.
1276         for (unsigned i = m_insts.size(); i--;) {
1277             for (Inst& inst : m_insts[i])
1278                 target->appendInst(WTFMove(inst));
1279         }
1280         m_insts.shrink(0);
1281     }
1282     
1283     Air::BasicBlock* newBlock()
1284     {
1285         return m_blockInsertionSet.insertAfter(m_blockToBlock[m_block]);
1286     }
1287
1288     // NOTE: This will create a continuation block (`nextBlock`) *after* any blocks you've created using
1289     // newBlock(). So, it's preferable to create all of your blocks upfront using newBlock(). Also note
1290     // that any code you emit before this will be prepended to the continuation, and any code you emit
1291     // after this will be appended to the previous block.
1292     void splitBlock(Air::BasicBlock*& previousBlock, Air::BasicBlock*& nextBlock)
1293     {
1294         Air::BasicBlock* block = m_blockToBlock[m_block];
1295         
1296         previousBlock = block;
1297         nextBlock = m_blockInsertionSet.insertAfter(block);
1298         
1299         finishAppendingInstructions(nextBlock);
1300         nextBlock->successors() = block->successors();
1301         block->successors().clear();
1302         
1303         m_insts.append(Vector<Inst>());
1304     }
1305     
1306     template<typename T, typename... Arguments>
1307     T* ensureSpecial(T*& field, Arguments&&... arguments)
1308     {
1309         if (!field) {
1310             field = static_cast<T*>(
1311                 m_code.addSpecial(makeUnique<T>(std::forward<Arguments>(arguments)...)));
1312         }
1313         return field;
1314     }
1315
1316     template<typename... Arguments>
1317     CheckSpecial* ensureCheckSpecial(Arguments&&... arguments)
1318     {
1319         CheckSpecial::Key key(std::forward<Arguments>(arguments)...);
1320         auto result = m_checkSpecials.add(key, nullptr);
1321         return ensureSpecial(result.iterator->value, key);
1322     }
1323
1324     void fillStackmap(Inst& inst, StackmapValue* stackmap, unsigned numSkipped)
1325     {
1326         for (unsigned i = numSkipped; i < stackmap->numChildren(); ++i) {
1327             ConstrainedValue value = stackmap->constrainedChild(i);
1328
1329             Arg arg;
1330             switch (value.rep().kind()) {
1331             case ValueRep::WarmAny:
1332             case ValueRep::ColdAny:
1333             case ValueRep::LateColdAny:
1334                 if (imm(value.value()))
1335                     arg = imm(value.value());
1336                 else if (value.value()->hasInt64())
1337                     arg = Arg::bigImm(value.value()->asInt64());
1338                 else if (value.value()->hasDouble() && canBeInternal(value.value())) {
1339                     commitInternal(value.value());
1340                     arg = Arg::bigImm(bitwise_cast<int64_t>(value.value()->asDouble()));
1341                 } else if (value.value()->hasFloat() && canBeInternal(value.value())) {
1342                     commitInternal(value.value());
1343                     arg = Arg::bigImm(static_cast<uint64_t>(bitwise_cast<uint32_t>(value.value()->asFloat())));
1344                 } else
1345                     arg = tmp(value.value());
1346                 break;
1347             case ValueRep::SomeRegister:
1348             case ValueRep::SomeLateRegister:
1349                 arg = tmp(value.value());
1350                 break;
1351             case ValueRep::SomeRegisterWithClobber: {
1352                 Tmp dstTmp = m_code.newTmp(value.value()->resultBank());
1353                 append(relaxedMoveForType(value.value()->type()), immOrTmp(value.value()), dstTmp);
1354                 arg = dstTmp;
1355                 break;
1356             }
1357             case ValueRep::LateRegister:
1358             case ValueRep::Register:
1359                 stackmap->earlyClobbered().clear(value.rep().reg());
1360                 arg = Tmp(value.rep().reg());
1361                 append(relaxedMoveForType(value.value()->type()), immOrTmp(value.value()), arg);
1362                 break;
1363             case ValueRep::StackArgument:
1364                 arg = Arg::callArg(value.rep().offsetFromSP());
1365                 append(trappingInst(m_value, createStore(moveForType(value.value()->type()), value.value(), arg)));
1366                 break;
1367             default:
1368                 RELEASE_ASSERT_NOT_REACHED();
1369                 break;
1370             }
1371             inst.args.append(arg);
1372         }
1373     }
1374     
1375     // Create an Inst to do the comparison specified by the given value.
1376     template<typename CompareFunctor, typename TestFunctor, typename CompareDoubleFunctor, typename CompareFloatFunctor>
1377     Inst createGenericCompare(
1378         Value* value,
1379         const CompareFunctor& compare, // Signature: (Width, Arg relCond, Arg, Arg) -> Inst
1380         const TestFunctor& test, // Signature: (Width, Arg resCond, Arg, Arg) -> Inst
1381         const CompareDoubleFunctor& compareDouble, // Signature: (Arg doubleCond, Arg, Arg) -> Inst
1382         const CompareFloatFunctor& compareFloat, // Signature: (Arg doubleCond, Arg, Arg) -> Inst
1383         bool inverted = false)
1384     {
1385         // NOTE: This is totally happy to match comparisons that have already been computed elsewhere
1386         // since on most architectures, the cost of branching on a previously computed comparison
1387         // result is almost always higher than just doing another fused compare/branch. The only time
1388         // it could be worse is if we have a binary comparison and both operands are variables (not
1389         // constants), and we encounter register pressure. Even in this case, duplicating the compare
1390         // so that we can fuse it to the branch will be more efficient most of the time, since
1391         // register pressure is not *that* common. For this reason, this algorithm will always
1392         // duplicate the comparison.
1393         //
1394         // However, we cannot duplicate loads. The canBeInternal() on a load will assume that we
1395         // already validated canBeInternal() on all of the values that got us to the load. So, even
1396         // if we are sharing a value, we still need to call canBeInternal() for the purpose of
1397         // tracking whether we are still in good shape to fuse loads.
1398         //
1399         // We could even have a chain of compare values that we fuse, and any member of the chain
1400         // could be shared. Once any of them are shared, then the shared one's transitive children
1401         // cannot be locked (i.e. commitInternal()). But if none of them are shared, then we want to
1402         // lock all of them because that's a prerequisite to fusing the loads so that the loads don't
1403         // get duplicated. For example, we might have: 
1404         //
1405         //     @tmp1 = LessThan(@a, @b)
1406         //     @tmp2 = Equal(@tmp1, 0)
1407         //     Branch(@tmp2)
1408         //
1409         // If either @a or @b are loads, then we want to have locked @tmp1 and @tmp2 so that they
1410         // don't emit the loads a second time. But if we had another use of @tmp2, then we cannot
1411         // lock @tmp1 (or @a or @b) because then we'll get into trouble when the other values that
1412         // try to share @tmp1 with us try to do their lowering.
1413         //
1414         // There's one more wrinkle. If we don't lock an internal value, then this internal value may
1415         // have already separately locked its children. So, if we're not locking a value then we need
1416         // to make sure that its children aren't locked. We encapsulate this in two ways:
1417         //
1418         // canCommitInternal: This variable tells us if the values that we've fused so far are
1419         // locked. This means that we're not sharing any of them with anyone. This permits us to fuse
1420         // loads. If it's false, then we cannot fuse loads and we also need to ensure that the
1421         // children of any values we try to fuse-by-sharing are not already locked. You don't have to
1422         // worry about the children locking thing if you use prepareToFuse() before trying to fuse a
1423         // sharable value. But, you do need to guard any load fusion by checking if canCommitInternal
1424         // is true.
1425         //
1426         // FusionResult prepareToFuse(value): Call this when you think that you would like to fuse
1427         // some value and that value is not a load. It will automatically handle the shared-or-locked
1428         // issues and it will clear canCommitInternal if necessary. This will return CannotFuse
1429         // (which acts like false) if the value cannot be locked and its children are locked. That's
1430         // rare, but you just need to make sure that you do smart things when this happens (i.e. just
1431         // use the value rather than trying to fuse it). After you call prepareToFuse(), you can
1432         // still change your mind about whether you will actually fuse the value. If you do fuse it,
1433         // you need to call commitFusion(value, fusionResult).
1434         //
1435         // commitFusion(value, fusionResult): Handles calling commitInternal(value) if fusionResult
1436         // is FuseAndCommit.
1437         
1438         bool canCommitInternal = true;
1439
1440         enum FusionResult {
1441             CannotFuse,
1442             FuseAndCommit,
1443             Fuse
1444         };
1445         auto prepareToFuse = [&] (Value* value) -> FusionResult {
1446             if (value == m_value) {
1447                 // It's not actually internal. It's the root value. We're good to go.
1448                 return Fuse;
1449             }
1450
1451             if (canCommitInternal && canBeInternal(value)) {
1452                 // We are the only users of this value. This also means that the value's children
1453                 // could not have been locked, since we have now proved that m_value dominates value
1454                 // in the data flow graph. To only other way to value is from a user of m_value. If
1455                 // value's children are shared with others, then they could not have been locked
1456                 // because their use count is greater than 1. If they are only used from value, then
1457                 // in order for value's children to be locked, value would also have to be locked,
1458                 // and we just proved that it wasn't.
1459                 return FuseAndCommit;
1460             }
1461
1462             // We're going to try to share value with others. It's possible that some other basic
1463             // block had already emitted code for value and then matched over its children and then
1464             // locked them, in which case we just want to use value instead of duplicating it. So, we
1465             // validate the children. Note that this only arises in linear chains like:
1466             //
1467             //     BB#1:
1468             //         @1 = Foo(...)
1469             //         @2 = Bar(@1)
1470             //         Jump(#2)
1471             //     BB#2:
1472             //         @3 = Baz(@2)
1473             //
1474             // Notice how we could start by generating code for BB#1 and then decide to lock @1 when
1475             // generating code for @2, if we have some way of fusing Bar and Foo into a single
1476             // instruction. This is legal, since indeed @1 only has one user. The fact that @2 now
1477             // has a tmp (i.e. @2 is pinned), canBeInternal(@2) will return false, which brings us
1478             // here. In that case, we cannot match over @2 because then we'd hit a hazard if we end
1479             // up deciding not to fuse Foo into the fused Baz/Bar.
1480             //
1481             // Happily, there are only two places where this kind of child validation happens is in
1482             // rules that admit sharing, like this and effectiveAddress().
1483             //
1484             // N.B. We could probably avoid the need to do value locking if we committed to a well
1485             // chosen code generation order. For example, if we guaranteed that all of the users of
1486             // a value get generated before that value, then there's no way for the lowering of @3 to
1487             // see @1 locked. But we don't want to do that, since this is a greedy instruction
1488             // selector and so we want to be able to play with order.
1489             for (Value* child : value->children()) {
1490                 if (m_locked.contains(child))
1491                     return CannotFuse;
1492             }
1493
1494             // It's safe to share value, but since we're sharing, it means that we aren't locking it.
1495             // If we don't lock it, then fusing loads is off limits and all of value's children will
1496             // have to go through the sharing path as well. Fusing loads is off limits because the load
1497             // could already have been emitted elsehwere - so fusing it here would duplicate the load.
1498             // We don't consider that to be a legal optimization.
1499             canCommitInternal = false;
1500             
1501             return Fuse;
1502         };
1503
1504         auto commitFusion = [&] (Value* value, FusionResult result) {
1505             if (result == FuseAndCommit)
1506                 commitInternal(value);
1507         };
1508         
1509         // Chew through any inversions. This loop isn't necessary for comparisons and branches, but
1510         // we do need at least one iteration of it for Check.
1511         for (;;) {
1512             bool shouldInvert =
1513                 (value->opcode() == BitXor && value->child(1)->hasInt() && (value->child(1)->asInt() == 1) && value->child(0)->returnsBool())
1514                 || (value->opcode() == Equal && value->child(1)->isInt(0));
1515             if (!shouldInvert)
1516                 break;
1517
1518             FusionResult fusionResult = prepareToFuse(value);
1519             if (fusionResult == CannotFuse)
1520                 break;
1521             commitFusion(value, fusionResult);
1522             
1523             value = value->child(0);
1524             inverted = !inverted;
1525         }
1526
1527         auto createRelCond = [&] (
1528             MacroAssembler::RelationalCondition relationalCondition,
1529             MacroAssembler::DoubleCondition doubleCondition) {
1530             Arg relCond = Arg::relCond(relationalCondition).inverted(inverted);
1531             Arg doubleCond = Arg::doubleCond(doubleCondition).inverted(inverted);
1532             Value* left = value->child(0);
1533             Value* right = value->child(1);
1534
1535             if (value->child(0)->type().isInt()) {
1536                 Arg rightImm = imm(right);
1537
1538                 auto tryCompare = [&] (
1539                     Width width, ArgPromise&& left, ArgPromise&& right) -> Inst {
1540                     if (Inst result = compare(width, relCond, left, right))
1541                         return result;
1542                     if (Inst result = compare(width, relCond.flipped(), right, left))
1543                         return result;
1544                     return Inst();
1545                 };
1546
1547                 auto tryCompareLoadImm = [&] (
1548                     Width width, B3::Opcode loadOpcode, Arg::Signedness signedness) -> Inst {
1549                     if (rightImm && rightImm.isRepresentableAs(width, signedness)) {
1550                         if (Inst result = tryCompare(width, loadPromise(left, loadOpcode), rightImm)) {
1551                             commitInternal(left);
1552                             return result;
1553                         }
1554                     }
1555                     return Inst();
1556                 };
1557
1558                 Width width = value->child(0)->resultWidth();
1559                 
1560                 if (canCommitInternal) {
1561                     // First handle compares that involve fewer bits than B3's type system supports.
1562                     // This is pretty important. For example, we want this to be a single
1563                     // instruction:
1564                     //
1565                     //     @1 = Load8S(...)
1566                     //     @2 = Const32(...)
1567                     //     @3 = LessThan(@1, @2)
1568                     //     Branch(@3)
1569                 
1570                     if (relCond.isSignedCond()) {
1571                         if (Inst result = tryCompareLoadImm(Width8, Load8S, Arg::Signed))
1572                             return result;
1573                     }
1574                 
1575                     if (relCond.isUnsignedCond()) {
1576                         if (Inst result = tryCompareLoadImm(Width8, Load8Z, Arg::Unsigned))
1577                             return result;
1578                     }
1579
1580                     if (relCond.isSignedCond()) {
1581                         if (Inst result = tryCompareLoadImm(Width16, Load16S, Arg::Signed))
1582                             return result;
1583                     }
1584                 
1585                     if (relCond.isUnsignedCond()) {
1586                         if (Inst result = tryCompareLoadImm(Width16, Load16Z, Arg::Unsigned))
1587                             return result;
1588                     }
1589
1590                     // Now handle compares that involve a load and an immediate.
1591
1592                     if (Inst result = tryCompareLoadImm(width, Load, Arg::Signed))
1593                         return result;
1594
1595                     // Now handle compares that involve a load. It's not obvious that it's better to
1596                     // handle this before the immediate cases or not. Probably doesn't matter.
1597
1598                     if (Inst result = tryCompare(width, loadPromise(left), tmpPromise(right))) {
1599                         commitInternal(left);
1600                         return result;
1601                     }
1602                 
1603                     if (Inst result = tryCompare(width, tmpPromise(left), loadPromise(right))) {
1604                         commitInternal(right);
1605                         return result;
1606                     }
1607                 }
1608
1609                 // Now handle compares that involve an immediate and a tmp.
1610                 
1611                 if (rightImm && rightImm.isRepresentableAs<int32_t>()) {
1612                     if (Inst result = tryCompare(width, tmpPromise(left), rightImm))
1613                         return result;
1614                 }
1615
1616                 // Finally, handle comparison between tmps.
1617                 ArgPromise leftPromise = tmpPromise(left);
1618                 ArgPromise rightPromise = tmpPromise(right);
1619                 return compare(width, relCond, leftPromise, rightPromise);
1620             }
1621
1622             // Floating point comparisons can't really do anything smart.
1623             ArgPromise leftPromise = tmpPromise(left);
1624             ArgPromise rightPromise = tmpPromise(right);
1625             if (value->child(0)->type() == Float)
1626                 return compareFloat(doubleCond, leftPromise, rightPromise);
1627             return compareDouble(doubleCond, leftPromise, rightPromise);
1628         };
1629
1630         Width width = value->resultWidth();
1631         Arg resCond = Arg::resCond(MacroAssembler::NonZero).inverted(inverted);
1632         
1633         auto tryTest = [&] (
1634             Width width, ArgPromise&& left, ArgPromise&& right) -> Inst {
1635             if (Inst result = test(width, resCond, left, right))
1636                 return result;
1637             if (Inst result = test(width, resCond, right, left))
1638                 return result;
1639             return Inst();
1640         };
1641
1642         auto attemptFused = [&] () -> Inst {
1643             switch (value->opcode()) {
1644             case NotEqual:
1645                 return createRelCond(MacroAssembler::NotEqual, MacroAssembler::DoubleNotEqualOrUnordered);
1646             case Equal:
1647                 return createRelCond(MacroAssembler::Equal, MacroAssembler::DoubleEqual);
1648             case LessThan:
1649                 return createRelCond(MacroAssembler::LessThan, MacroAssembler::DoubleLessThan);
1650             case GreaterThan:
1651                 return createRelCond(MacroAssembler::GreaterThan, MacroAssembler::DoubleGreaterThan);
1652             case LessEqual:
1653                 return createRelCond(MacroAssembler::LessThanOrEqual, MacroAssembler::DoubleLessThanOrEqual);
1654             case GreaterEqual:
1655                 return createRelCond(MacroAssembler::GreaterThanOrEqual, MacroAssembler::DoubleGreaterThanOrEqual);
1656             case EqualOrUnordered:
1657                 // The integer condition is never used in this case.
1658                 return createRelCond(MacroAssembler::Equal, MacroAssembler::DoubleEqualOrUnordered);
1659             case Above:
1660                 // We use a bogus double condition because these integer comparisons won't got down that
1661                 // path anyway.
1662                 return createRelCond(MacroAssembler::Above, MacroAssembler::DoubleEqual);
1663             case Below:
1664                 return createRelCond(MacroAssembler::Below, MacroAssembler::DoubleEqual);
1665             case AboveEqual:
1666                 return createRelCond(MacroAssembler::AboveOrEqual, MacroAssembler::DoubleEqual);
1667             case BelowEqual:
1668                 return createRelCond(MacroAssembler::BelowOrEqual, MacroAssembler::DoubleEqual);
1669             case BitAnd: {
1670                 Value* left = value->child(0);
1671                 Value* right = value->child(1);
1672
1673                 bool hasRightConst;
1674                 int64_t rightConst;
1675                 Arg rightImm;
1676                 Arg rightImm64;
1677
1678                 hasRightConst = right->hasInt();
1679                 if (hasRightConst) {
1680                     rightConst = right->asInt();
1681                     rightImm = bitImm(right);
1682                     rightImm64 = bitImm64(right);
1683                 }
1684                 
1685                 auto tryTestLoadImm = [&] (Width width, Arg::Signedness signedness, B3::Opcode loadOpcode) -> Inst {
1686                     if (!hasRightConst)
1687                         return Inst();
1688                     // Signed loads will create high bits, so if the immediate has high bits
1689                     // then we cannot proceed. Consider BitAnd(Load8S(ptr), 0x101). This cannot
1690                     // be turned into testb (ptr), $1, since if the high bit within that byte
1691                     // was set then it would be extended to include 0x100. The handling below
1692                     // won't anticipate this, so we need to catch it here.
1693                     if (signedness == Arg::Signed
1694                         && !Arg::isRepresentableAs(width, Arg::Unsigned, rightConst))
1695                         return Inst();
1696                     
1697                     // FIXME: If this is unsigned then we can chop things off of the immediate.
1698                     // This might make the immediate more legal. Perhaps that's a job for
1699                     // strength reduction?
1700                     // https://bugs.webkit.org/show_bug.cgi?id=169248
1701                     
1702                     if (rightImm) {
1703                         if (Inst result = tryTest(width, loadPromise(left, loadOpcode), rightImm)) {
1704                             commitInternal(left);
1705                             return result;
1706                         }
1707                     }
1708                     if (rightImm64) {
1709                         if (Inst result = tryTest(width, loadPromise(left, loadOpcode), rightImm64)) {
1710                             commitInternal(left);
1711                             return result;
1712                         }
1713                     }
1714                     return Inst();
1715                 };
1716
1717                 if (canCommitInternal) {
1718                     // First handle test's that involve fewer bits than B3's type system supports.
1719
1720                     if (Inst result = tryTestLoadImm(Width8, Arg::Unsigned, Load8Z))
1721                         return result;
1722                     
1723                     if (Inst result = tryTestLoadImm(Width8, Arg::Signed, Load8S))
1724                         return result;
1725                     
1726                     if (Inst result = tryTestLoadImm(Width16, Arg::Unsigned, Load16Z))
1727                         return result;
1728                     
1729                     if (Inst result = tryTestLoadImm(Width16, Arg::Signed, Load16S))
1730                         return result;
1731
1732                     // This allows us to use a 32-bit test for 64-bit BitAnd if the immediate is
1733                     // representable as an unsigned 32-bit value. The logic involved is the same
1734                     // as if we were pondering using a 32-bit test for
1735                     // BitAnd(SExt(Load(ptr)), const), in the sense that in both cases we have
1736                     // to worry about high bits. So, we use the "Signed" version of this helper.
1737                     if (Inst result = tryTestLoadImm(Width32, Arg::Signed, Load))
1738                         return result;
1739                     
1740                     // This is needed to handle 32-bit test for arbitrary 32-bit immediates.
1741                     if (Inst result = tryTestLoadImm(width, Arg::Unsigned, Load))
1742                         return result;
1743                     
1744                     // Now handle test's that involve a load.
1745                     
1746                     Width width = value->child(0)->resultWidth();
1747                     if (Inst result = tryTest(width, loadPromise(left), tmpPromise(right))) {
1748                         commitInternal(left);
1749                         return result;
1750                     }
1751                     
1752                     if (Inst result = tryTest(width, tmpPromise(left), loadPromise(right))) {
1753                         commitInternal(right);
1754                         return result;
1755                     }
1756                 }
1757
1758                 // Now handle test's that involve an immediate and a tmp.
1759
1760                 if (hasRightConst) {
1761                     if ((width == Width32 && rightConst == 0xffffffff)
1762                         || (width == Width64 && rightConst == -1)) {
1763                         if (Inst result = tryTest(width, tmpPromise(left), tmpPromise(left)))
1764                             return result;
1765                     }
1766                     if (isRepresentableAs<uint32_t>(rightConst)) {
1767                         if (Inst result = tryTest(Width32, tmpPromise(left), rightImm))
1768                             return result;
1769                         if (Inst result = tryTest(Width32, tmpPromise(left), rightImm64))
1770                             return result;
1771                     }
1772                     if (Inst result = tryTest(width, tmpPromise(left), rightImm))
1773                         return result;
1774                     if (Inst result = tryTest(width, tmpPromise(left), rightImm64))
1775                         return result;
1776                 }
1777
1778                 // Finally, just do tmp's.
1779                 return tryTest(width, tmpPromise(left), tmpPromise(right));
1780             }
1781             default:
1782                 return Inst();
1783             }
1784         };
1785
1786         if (FusionResult fusionResult = prepareToFuse(value)) {
1787             if (Inst result = attemptFused()) {
1788                 commitFusion(value, fusionResult);
1789                 return result;
1790             }
1791         }
1792
1793         if (Arg::isValidBitImmForm(-1)) {
1794             if (canCommitInternal && value->as<MemoryValue>()) {
1795                 // Handle things like Branch(Load8Z(value))
1796
1797                 if (Inst result = tryTest(Width8, loadPromise(value, Load8Z), Arg::bitImm(-1))) {
1798                     commitInternal(value);
1799                     return result;
1800                 }
1801
1802                 if (Inst result = tryTest(Width8, loadPromise(value, Load8S), Arg::bitImm(-1))) {
1803                     commitInternal(value);
1804                     return result;
1805                 }
1806
1807                 if (Inst result = tryTest(Width16, loadPromise(value, Load16Z), Arg::bitImm(-1))) {
1808                     commitInternal(value);
1809                     return result;
1810                 }
1811
1812                 if (Inst result = tryTest(Width16, loadPromise(value, Load16S), Arg::bitImm(-1))) {
1813                     commitInternal(value);
1814                     return result;
1815                 }
1816
1817                 if (Inst result = tryTest(width, loadPromise(value), Arg::bitImm(-1))) {
1818                     commitInternal(value);
1819                     return result;
1820                 }
1821             }
1822
1823             ArgPromise leftPromise = tmpPromise(value);
1824             ArgPromise rightPromise = Arg::bitImm(-1);
1825             if (Inst result = test(width, resCond, leftPromise, rightPromise))
1826                 return result;
1827         }
1828         
1829         // Sometimes this is the only form of test available. We prefer not to use this because
1830         // it's less canonical.
1831         ArgPromise leftPromise = tmpPromise(value);
1832         ArgPromise rightPromise = tmpPromise(value);
1833         return test(width, resCond, leftPromise, rightPromise);
1834     }
1835
1836     Inst createBranch(Value* value, bool inverted = false)
1837     {
1838         using namespace Air;
1839         return createGenericCompare(
1840             value,
1841             [this] (
1842                 Width width, const Arg& relCond,
1843                 ArgPromise& left, ArgPromise& right) -> Inst {
1844                 switch (width) {
1845                 case Width8:
1846                     if (isValidForm(Branch8, Arg::RelCond, left.kind(), right.kind())) {
1847                         return left.inst(right.inst(
1848                             Branch8, m_value, relCond,
1849                             left.consume(*this), right.consume(*this)));
1850                     }
1851                     return Inst();
1852                 case Width16:
1853                     return Inst();
1854                 case Width32:
1855                     if (isValidForm(Branch32, Arg::RelCond, left.kind(), right.kind())) {
1856                         return left.inst(right.inst(
1857                             Branch32, m_value, relCond,
1858                             left.consume(*this), right.consume(*this)));
1859                     }
1860                     return Inst();
1861                 case Width64:
1862                     if (isValidForm(Branch64, Arg::RelCond, left.kind(), right.kind())) {
1863                         return left.inst(right.inst(
1864                             Branch64, m_value, relCond,
1865                             left.consume(*this), right.consume(*this)));
1866                     }
1867                     return Inst();
1868                 }
1869                 ASSERT_NOT_REACHED();
1870             },
1871             [this] (
1872                 Width width, const Arg& resCond,
1873                 ArgPromise& left, ArgPromise& right) -> Inst {
1874                 switch (width) {
1875                 case Width8:
1876                     if (isValidForm(BranchTest8, Arg::ResCond, left.kind(), right.kind())) {
1877                         return left.inst(right.inst(
1878                             BranchTest8, m_value, resCond,
1879                             left.consume(*this), right.consume(*this)));
1880                     }
1881                     return Inst();
1882                 case Width16:
1883                     return Inst();
1884                 case Width32:
1885                     if (isValidForm(BranchTest32, Arg::ResCond, left.kind(), right.kind())) {
1886                         return left.inst(right.inst(
1887                             BranchTest32, m_value, resCond,
1888                             left.consume(*this), right.consume(*this)));
1889                     }
1890                     return Inst();
1891                 case Width64:
1892                     if (isValidForm(BranchTest64, Arg::ResCond, left.kind(), right.kind())) {
1893                         return left.inst(right.inst(
1894                             BranchTest64, m_value, resCond,
1895                             left.consume(*this), right.consume(*this)));
1896                     }
1897                     return Inst();
1898                 }
1899                 ASSERT_NOT_REACHED();
1900             },
1901             [this] (Arg doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
1902                 if (isValidForm(BranchDouble, Arg::DoubleCond, left.kind(), right.kind())) {
1903                     return left.inst(right.inst(
1904                         BranchDouble, m_value, doubleCond,
1905                         left.consume(*this), right.consume(*this)));
1906                 }
1907                 return Inst();
1908             },
1909             [this] (Arg doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
1910                 if (isValidForm(BranchFloat, Arg::DoubleCond, left.kind(), right.kind())) {
1911                     return left.inst(right.inst(
1912                         BranchFloat, m_value, doubleCond,
1913                         left.consume(*this), right.consume(*this)));
1914                 }
1915                 return Inst();
1916             },
1917             inverted);
1918     }
1919
1920     Inst createCompare(Value* value, bool inverted = false)
1921     {
1922         using namespace Air;
1923         return createGenericCompare(
1924             value,
1925             [this] (
1926                 Width width, const Arg& relCond,
1927                 ArgPromise& left, ArgPromise& right) -> Inst {
1928                 switch (width) {
1929                 case Width8:
1930                 case Width16:
1931                     return Inst();
1932                 case Width32:
1933                     if (isValidForm(Compare32, Arg::RelCond, left.kind(), right.kind(), Arg::Tmp)) {
1934                         return left.inst(right.inst(
1935                             Compare32, m_value, relCond,
1936                             left.consume(*this), right.consume(*this), tmp(m_value)));
1937                     }
1938                     return Inst();
1939                 case Width64:
1940                     if (isValidForm(Compare64, Arg::RelCond, left.kind(), right.kind(), Arg::Tmp)) {
1941                         return left.inst(right.inst(
1942                             Compare64, m_value, relCond,
1943                             left.consume(*this), right.consume(*this), tmp(m_value)));
1944                     }
1945                     return Inst();
1946                 }
1947                 ASSERT_NOT_REACHED();
1948             },
1949             [this] (
1950                 Width width, const Arg& resCond,
1951                 ArgPromise& left, ArgPromise& right) -> Inst {
1952                 switch (width) {
1953                 case Width8:
1954                 case Width16:
1955                     return Inst();
1956                 case Width32:
1957                     if (isValidForm(Test32, Arg::ResCond, left.kind(), right.kind(), Arg::Tmp)) {
1958                         return left.inst(right.inst(
1959                             Test32, m_value, resCond,
1960                             left.consume(*this), right.consume(*this), tmp(m_value)));
1961                     }
1962                     return Inst();
1963                 case Width64:
1964                     if (isValidForm(Test64, Arg::ResCond, left.kind(), right.kind(), Arg::Tmp)) {
1965                         return left.inst(right.inst(
1966                             Test64, m_value, resCond,
1967                             left.consume(*this), right.consume(*this), tmp(m_value)));
1968                     }
1969                     return Inst();
1970                 }
1971                 ASSERT_NOT_REACHED();
1972             },
1973             [this] (const Arg& doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
1974                 if (isValidForm(CompareDouble, Arg::DoubleCond, left.kind(), right.kind(), Arg::Tmp)) {
1975                     return left.inst(right.inst(
1976                         CompareDouble, m_value, doubleCond,
1977                         left.consume(*this), right.consume(*this), tmp(m_value)));
1978                 }
1979                 return Inst();
1980             },
1981             [this] (const Arg& doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
1982                 if (isValidForm(CompareFloat, Arg::DoubleCond, left.kind(), right.kind(), Arg::Tmp)) {
1983                     return left.inst(right.inst(
1984                         CompareFloat, m_value, doubleCond,
1985                         left.consume(*this), right.consume(*this), tmp(m_value)));
1986                 }
1987                 return Inst();
1988             },
1989             inverted);
1990     }
1991
1992     struct MoveConditionallyConfig {
1993         Air::Opcode moveConditionally32;
1994         Air::Opcode moveConditionally64;
1995         Air::Opcode moveConditionallyTest32;
1996         Air::Opcode moveConditionallyTest64;
1997         Air::Opcode moveConditionallyDouble;
1998         Air::Opcode moveConditionallyFloat;
1999     };
2000     Inst createSelect(const MoveConditionallyConfig& config)
2001     {
2002         using namespace Air;
2003         auto createSelectInstruction = [&] (Air::Opcode opcode, const Arg& condition, ArgPromise& left, ArgPromise& right) -> Inst {
2004             if (isValidForm(opcode, condition.kind(), left.kind(), right.kind(), Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2005                 Tmp result = tmp(m_value);
2006                 Tmp thenCase = tmp(m_value->child(1));
2007                 Tmp elseCase = tmp(m_value->child(2));
2008                 return left.inst(right.inst(
2009                     opcode, m_value, condition,
2010                     left.consume(*this), right.consume(*this), thenCase, elseCase, result));
2011             }
2012             if (isValidForm(opcode, condition.kind(), left.kind(), right.kind(), Arg::Tmp, Arg::Tmp)) {
2013                 Tmp result = tmp(m_value);
2014                 Tmp source = tmp(m_value->child(1));
2015                 append(relaxedMoveForType(m_value->type()), tmp(m_value->child(2)), result);
2016                 return left.inst(right.inst(
2017                     opcode, m_value, condition,
2018                     left.consume(*this), right.consume(*this), source, result));
2019             }
2020             return Inst();
2021         };
2022
2023         return createGenericCompare(
2024             m_value->child(0),
2025             [&] (Width width, const Arg& relCond, ArgPromise& left, ArgPromise& right) -> Inst {
2026                 switch (width) {
2027                 case Width8:
2028                     // FIXME: Support these things.
2029                     // https://bugs.webkit.org/show_bug.cgi?id=151504
2030                     return Inst();
2031                 case Width16:
2032                     return Inst();
2033                 case Width32:
2034                     return createSelectInstruction(config.moveConditionally32, relCond, left, right);
2035                 case Width64:
2036                     return createSelectInstruction(config.moveConditionally64, relCond, left, right);
2037                 }
2038                 ASSERT_NOT_REACHED();
2039             },
2040             [&] (Width width, const Arg& resCond, ArgPromise& left, ArgPromise& right) -> Inst {
2041                 switch (width) {
2042                 case Width8:
2043                     // FIXME: Support more things.
2044                     // https://bugs.webkit.org/show_bug.cgi?id=151504
2045                     return Inst();
2046                 case Width16:
2047                     return Inst();
2048                 case Width32:
2049                     return createSelectInstruction(config.moveConditionallyTest32, resCond, left, right);
2050                 case Width64:
2051                     return createSelectInstruction(config.moveConditionallyTest64, resCond, left, right);
2052                 }
2053                 ASSERT_NOT_REACHED();
2054             },
2055             [&] (Arg doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
2056                 return createSelectInstruction(config.moveConditionallyDouble, doubleCond, left, right);
2057             },
2058             [&] (Arg doubleCond, ArgPromise& left, ArgPromise& right) -> Inst {
2059                 return createSelectInstruction(config.moveConditionallyFloat, doubleCond, left, right);
2060             },
2061             false);
2062     }
2063     
2064     bool tryAppendLea()
2065     {
2066         using namespace Air;
2067         Air::Opcode leaOpcode = tryOpcodeForType(Lea32, Lea64, m_value->type());
2068         if (!isValidForm(leaOpcode, Arg::Index, Arg::Tmp))
2069             return false;
2070         
2071         // This lets us turn things like this:
2072         //
2073         //     Add(Add(@x, Shl(@y, $2)), $100)
2074         //
2075         // Into this:
2076         //
2077         //     lea 100(%rdi,%rsi,4), %rax
2078         //
2079         // We have a choice here between committing the internal bits of an index or sharing
2080         // them. There are solid arguments for both.
2081         //
2082         // Sharing: The word on the street is that the cost of a lea is one cycle no matter
2083         // what it does. Every experiment I've ever seen seems to confirm this. So, sharing
2084         // helps us in situations where Wasm input did this:
2085         //
2086         //     x = a[i].x;
2087         //     y = a[i].y;
2088         //
2089         // With sharing we would do:
2090         //
2091         //     leal (%a,%i,4), %tmp
2092         //     cmp (%size, %tmp)
2093         //     ja _fail
2094         //     movl (%base, %tmp), %x
2095         //     leal 4(%a,%i,4), %tmp
2096         //     cmp (%size, %tmp)
2097         //     ja _fail
2098         //     movl (%base, %tmp), %y
2099         //
2100         // In the absence of sharing, we may find ourselves needing separate registers for
2101         // the innards of the index. That's relatively unlikely to be a thing due to other
2102         // optimizations that we already have, but it could happen
2103         //
2104         // Committing: The worst case is that there is a complicated graph of additions and
2105         // shifts, where each value has multiple uses. In that case, it's better to compute
2106         // each one separately from the others since that way, each calculation will use a
2107         // relatively nearby tmp as its input. That seems uncommon, but in those cases,
2108         // committing is a clear winner: it would result in a simple interference graph
2109         // while sharing would result in a complex one. Interference sucks because it means
2110         // more time in IRC and it means worse code.
2111         //
2112         // It's not super clear if any of these corner cases would ever arise. Committing
2113         // has the benefit that it's easier to reason about, and protects a much darker
2114         // corner case (more interference).
2115                 
2116         // Here are the things we want to match:
2117         // Add(Add(@x, @y), $c)
2118         // Add(Shl(@x, $c), @y)
2119         // Add(@x, Shl(@y, $c))
2120         // Add(Add(@x, Shl(@y, $c)), $d)
2121         // Add(Add(Shl(@x, $c), @y), $d)
2122         //
2123         // Note that if you do Add(Shl(@x, $c), $d) then we will treat $d as a non-constant and
2124         // force it to materialize. You'll get something like this:
2125         //
2126         // movl $d, %tmp
2127         // leal (%tmp,%x,1<<c), %result
2128         //
2129         // Which is pretty close to optimal and has the nice effect of being able to handle large
2130         // constants gracefully.
2131         
2132         Value* innerAdd = nullptr;
2133         
2134         Value* value = m_value;
2135         
2136         // We're going to consume Add(Add(_), $c). If we succeed at consuming it then we have these
2137         // patterns left (i.e. in the Add(_)):
2138         //
2139         // Add(Add(@x, @y), $c)
2140         // Add(Add(@x, Shl(@y, $c)), $d)
2141         // Add(Add(Shl(@x, $c), @y), $d)
2142         //
2143         // Otherwise we are looking at these patterns:
2144         //
2145         // Add(Shl(@x, $c), @y)
2146         // Add(@x, Shl(@y, $c))
2147         //
2148         // This means that the subsequent code only has to worry about three patterns:
2149         //
2150         // Add(Shl(@x, $c), @y)
2151         // Add(@x, Shl(@y, $c))
2152         // Add(@x, @y) (only if offset != 0)
2153         Value::OffsetType offset = 0;
2154         if (value->child(1)->isRepresentableAs<Value::OffsetType>()
2155             && canBeInternal(value->child(0))
2156             && value->child(0)->opcode() == Add) {
2157             innerAdd = value->child(0);
2158             offset = static_cast<Value::OffsetType>(value->child(1)->asInt());
2159             value = value->child(0);
2160         }
2161         
2162         auto tryShl = [&] (Value* shl, Value* other) -> bool {
2163             Optional<unsigned> scale = scaleForShl(shl, offset);
2164             if (!scale)
2165                 return false;
2166             if (!canBeInternal(shl))
2167                 return false;
2168             
2169             ASSERT(!m_locked.contains(shl->child(0)));
2170             ASSERT(!m_locked.contains(other));
2171             
2172             append(leaOpcode, Arg::index(tmp(other), tmp(shl->child(0)), *scale, offset), tmp(m_value));
2173             commitInternal(innerAdd);
2174             commitInternal(shl);
2175             return true;
2176         };
2177         
2178         if (tryShl(value->child(0), value->child(1)))
2179             return true;
2180         if (tryShl(value->child(1), value->child(0)))
2181             return true;
2182         
2183         // The remaining pattern is just:
2184         // Add(@x, @y) (only if offset != 0)
2185         if (!offset)
2186             return false;
2187         ASSERT(!m_locked.contains(value->child(0)));
2188         ASSERT(!m_locked.contains(value->child(1)));
2189         append(leaOpcode, Arg::index(tmp(value->child(0)), tmp(value->child(1)), 1, offset), tmp(m_value));
2190         commitInternal(innerAdd);
2191         return true;
2192     }
2193
2194     void appendX86Div(B3::Opcode op)
2195     {
2196         using namespace Air;
2197         Air::Opcode convertToDoubleWord;
2198         Air::Opcode div;
2199         switch (m_value->type().kind()) {
2200         case Int32:
2201             convertToDoubleWord = X86ConvertToDoubleWord32;
2202             div = X86Div32;
2203             break;
2204         case Int64:
2205             convertToDoubleWord = X86ConvertToQuadWord64;
2206             div = X86Div64;
2207             break;
2208         default:
2209             RELEASE_ASSERT_NOT_REACHED();
2210             return;
2211         }
2212
2213         ASSERT(op == Div || op == Mod);
2214         Tmp result = op == Div ? m_eax : m_edx;
2215
2216         append(Move, tmp(m_value->child(0)), m_eax);
2217         append(convertToDoubleWord, m_eax, m_edx);
2218         append(div, m_eax, m_edx, tmp(m_value->child(1)));
2219         append(Move, result, tmp(m_value));
2220     }
2221
2222     void appendX86UDiv(B3::Opcode op)
2223     {
2224         using namespace Air;
2225         Air::Opcode div = m_value->type() == Int32 ? X86UDiv32 : X86UDiv64;
2226
2227         ASSERT(op == UDiv || op == UMod);
2228         Tmp result = op == UDiv ? m_eax : m_edx;
2229
2230         append(Move, tmp(m_value->child(0)), m_eax);
2231         append(Xor64, m_edx, m_edx);
2232         append(div, m_eax, m_edx, tmp(m_value->child(1)));
2233         append(Move, result, tmp(m_value));
2234     }
2235     
2236     Air::Opcode loadLinkOpcode(Width width, bool fence)
2237     {
2238         return fence ? OPCODE_FOR_WIDTH(LoadLinkAcq, width) : OPCODE_FOR_WIDTH(LoadLink, width);
2239     }
2240     
2241     Air::Opcode storeCondOpcode(Width width, bool fence)
2242     {
2243         return fence ? OPCODE_FOR_WIDTH(StoreCondRel, width) : OPCODE_FOR_WIDTH(StoreCond, width);
2244     }
2245     
2246     // This can emit code for the following patterns:
2247     // AtomicWeakCAS
2248     // BitXor(AtomicWeakCAS, 1)
2249     // AtomicStrongCAS
2250     // Equal(AtomicStrongCAS, expected)
2251     // NotEqual(AtomicStrongCAS, expected)
2252     // Branch(AtomicWeakCAS)
2253     // Branch(Equal(AtomicStrongCAS, expected))
2254     // Branch(NotEqual(AtomicStrongCAS, expected))
2255     //
2256     // It assumes that atomicValue points to the CAS, and m_value points to the instruction being
2257     // generated. It assumes that you've consumed everything that needs to be consumed.
2258     void appendCAS(Value* atomicValue, bool invert)
2259     {
2260         using namespace Air;
2261         AtomicValue* atomic = atomicValue->as<AtomicValue>();
2262         RELEASE_ASSERT(atomic);
2263         
2264         bool isBranch = m_value->opcode() == Branch;
2265         bool isStrong = atomic->opcode() == AtomicStrongCAS;
2266         bool returnsOldValue = m_value->opcode() == AtomicStrongCAS;
2267         bool hasFence = atomic->hasFence();
2268         
2269         Width width = atomic->accessWidth();
2270         Arg address = addr(atomic);
2271
2272         Tmp valueResultTmp;
2273         Tmp boolResultTmp;
2274         if (returnsOldValue) {
2275             RELEASE_ASSERT(!invert);
2276             valueResultTmp = tmp(m_value);
2277             boolResultTmp = m_code.newTmp(GP);
2278         } else if (isBranch) {
2279             valueResultTmp = m_code.newTmp(GP);
2280             boolResultTmp = m_code.newTmp(GP);
2281         } else {
2282             valueResultTmp = m_code.newTmp(GP);
2283             boolResultTmp = tmp(m_value);
2284         }
2285         
2286         Tmp successBoolResultTmp;
2287         if (isStrong && !isBranch)
2288             successBoolResultTmp = m_code.newTmp(GP);
2289         else
2290             successBoolResultTmp = boolResultTmp;
2291
2292         Tmp expectedValueTmp = tmp(atomic->child(0));
2293         Tmp newValueTmp = tmp(atomic->child(1));
2294         
2295         Air::FrequentedBlock success;
2296         Air::FrequentedBlock failure;
2297         if (isBranch) {
2298             success = m_blockToBlock[m_block]->successor(invert);
2299             failure = m_blockToBlock[m_block]->successor(!invert);
2300         }
2301         
2302         if (isX86()) {
2303             append(relaxedMoveForType(atomic->accessType()), immOrTmp(atomic->child(0)), m_eax);
2304             if (returnsOldValue) {
2305                 appendTrapping(OPCODE_FOR_WIDTH(AtomicStrongCAS, width), m_eax, newValueTmp, address);
2306                 append(relaxedMoveForType(atomic->accessType()), m_eax, valueResultTmp);
2307             } else if (isBranch) {
2308                 appendTrapping(OPCODE_FOR_WIDTH(BranchAtomicStrongCAS, width), Arg::statusCond(MacroAssembler::Success), m_eax, newValueTmp, address);
2309                 m_blockToBlock[m_block]->setSuccessors(success, failure);
2310             } else
2311                 appendTrapping(OPCODE_FOR_WIDTH(AtomicStrongCAS, width), Arg::statusCond(invert ? MacroAssembler::Failure : MacroAssembler::Success), m_eax, tmp(atomic->child(1)), address, boolResultTmp);
2312             return;
2313         }
2314         
2315         RELEASE_ASSERT(isARM64());
2316         // We wish to emit:
2317         //
2318         // Block #reloop:
2319         //     LoadLink
2320         //     Branch NotEqual
2321         //   Successors: Then:#fail, Else: #store
2322         // Block #store:
2323         //     StoreCond
2324         //     Xor $1, %result    <--- only if !invert
2325         //     Jump
2326         //   Successors: #done
2327         // Block #fail:
2328         //     Move $invert, %result
2329         //     Jump
2330         //   Successors: #done
2331         // Block #done:
2332         
2333         Air::BasicBlock* reloopBlock = newBlock();
2334         Air::BasicBlock* storeBlock = newBlock();
2335         Air::BasicBlock* successBlock = nullptr;
2336         if (!isBranch && isStrong)
2337             successBlock = newBlock();
2338         Air::BasicBlock* failBlock = nullptr;
2339         if (!isBranch) {
2340             failBlock = newBlock();
2341             failure = failBlock;
2342         }
2343         Air::BasicBlock* strongFailBlock;
2344         if (isStrong && hasFence)
2345             strongFailBlock = newBlock();
2346         Air::FrequentedBlock comparisonFail = failure;
2347         Air::FrequentedBlock weakFail;
2348         if (isStrong) {
2349             if (hasFence)
2350                 comparisonFail = strongFailBlock;
2351             weakFail = reloopBlock;
2352         } else 
2353             weakFail = failure;
2354         Air::BasicBlock* beginBlock;
2355         Air::BasicBlock* doneBlock;
2356         splitBlock(beginBlock, doneBlock);
2357         
2358         append(Air::Jump);
2359         beginBlock->setSuccessors(reloopBlock);
2360         
2361         reloopBlock->append(trappingInst(m_value, loadLinkOpcode(width, atomic->hasFence()), m_value, address, valueResultTmp));
2362         reloopBlock->append(OPCODE_FOR_CANONICAL_WIDTH(Branch, width), m_value, Arg::relCond(MacroAssembler::NotEqual), valueResultTmp, expectedValueTmp);
2363         reloopBlock->setSuccessors(comparisonFail, storeBlock);
2364         
2365         storeBlock->append(trappingInst(m_value, storeCondOpcode(width, atomic->hasFence()), m_value, newValueTmp, address, successBoolResultTmp));
2366         if (isBranch) {
2367             storeBlock->append(BranchTest32, m_value, Arg::resCond(MacroAssembler::Zero), boolResultTmp, boolResultTmp);
2368             storeBlock->setSuccessors(success, weakFail);
2369             doneBlock->successors().clear();
2370             RELEASE_ASSERT(!doneBlock->size());
2371             doneBlock->append(Air::Oops, m_value);
2372         } else {
2373             if (isStrong) {
2374                 storeBlock->append(BranchTest32, m_value, Arg::resCond(MacroAssembler::Zero), successBoolResultTmp, successBoolResultTmp);
2375                 storeBlock->setSuccessors(successBlock, reloopBlock);
2376                 
2377                 successBlock->append(Move, m_value, Arg::imm(!invert), boolResultTmp);
2378                 successBlock->append(Air::Jump, m_value);
2379                 successBlock->setSuccessors(doneBlock);
2380             } else {
2381                 if (!invert)
2382                     storeBlock->append(Xor32, m_value, Arg::bitImm(1), boolResultTmp, boolResultTmp);
2383                 
2384                 storeBlock->append(Air::Jump, m_value);
2385                 storeBlock->setSuccessors(doneBlock);
2386             }
2387             
2388             failBlock->append(Move, m_value, Arg::imm(invert), boolResultTmp);
2389             failBlock->append(Air::Jump, m_value);
2390             failBlock->setSuccessors(doneBlock);
2391         }
2392         
2393         if (isStrong && hasFence) {
2394             Tmp tmp = m_code.newTmp(GP);
2395             strongFailBlock->append(trappingInst(m_value, storeCondOpcode(width, atomic->hasFence()), m_value, valueResultTmp, address, tmp));
2396             strongFailBlock->append(BranchTest32, m_value, Arg::resCond(MacroAssembler::Zero), tmp, tmp);
2397             strongFailBlock->setSuccessors(failure, reloopBlock);
2398         }
2399     }
2400     
2401     bool appendVoidAtomic(Air::Opcode atomicOpcode)
2402     {
2403         if (m_useCounts.numUses(m_value))
2404             return false;
2405         
2406         Arg address = addr(m_value);
2407         
2408         if (isValidForm(atomicOpcode, Arg::Imm, address.kind()) && imm(m_value->child(0))) {
2409             append(atomicOpcode, imm(m_value->child(0)), address);
2410             return true;
2411         }
2412         
2413         if (isValidForm(atomicOpcode, Arg::Tmp, address.kind())) {
2414             append(atomicOpcode, tmp(m_value->child(0)), address);
2415             return true;
2416         }
2417         
2418         return false;
2419     }
2420     
2421     void appendGeneralAtomic(Air::Opcode opcode, Commutativity commutativity = NotCommutative)
2422     {
2423         using namespace Air;
2424         AtomicValue* atomic = m_value->as<AtomicValue>();
2425         
2426         Arg address = addr(m_value);
2427         Tmp oldValue = m_code.newTmp(GP);
2428         Tmp newValue = opcode == Air::Nop ? tmp(atomic->child(0)) : m_code.newTmp(GP);
2429         
2430         // We need a CAS loop or a LL/SC loop. Using prepare/attempt jargon, we want:
2431         //
2432         // Block #reloop:
2433         //     Prepare
2434         //     opcode
2435         //     Attempt
2436         //   Successors: Then:#done, Else:#reloop
2437         // Block #done:
2438         //     Move oldValue, result
2439         
2440         append(relaxedMoveForType(atomic->type()), oldValue, tmp(atomic));
2441         
2442         Air::BasicBlock* reloopBlock = newBlock();
2443         Air::BasicBlock* beginBlock;
2444         Air::BasicBlock* doneBlock;
2445         splitBlock(beginBlock, doneBlock);
2446         
2447         append(Air::Jump);
2448         beginBlock->setSuccessors(reloopBlock);
2449         
2450         Air::Opcode prepareOpcode;
2451         if (isX86()) {
2452             switch (atomic->accessWidth()) {
2453             case Width8:
2454                 prepareOpcode = Load8SignedExtendTo32;
2455                 break;
2456             case Width16:
2457                 prepareOpcode = Load16SignedExtendTo32;
2458                 break;
2459             case Width32:
2460                 prepareOpcode = Move32;
2461                 break;
2462             case Width64:
2463                 prepareOpcode = Move;
2464                 break;
2465             }
2466         } else {
2467             RELEASE_ASSERT(isARM64());
2468             prepareOpcode = loadLinkOpcode(atomic->accessWidth(), atomic->hasFence());
2469         }
2470         reloopBlock->append(trappingInst(m_value, prepareOpcode, m_value, address, oldValue));
2471         
2472         if (opcode != Air::Nop) {
2473             // FIXME: If we ever have to write this again, we need to find a way to share the code with
2474             // appendBinOp.
2475             // https://bugs.webkit.org/show_bug.cgi?id=169249
2476             if (commutativity == Commutative && imm(atomic->child(0)) && isValidForm(opcode, Arg::Imm, Arg::Tmp, Arg::Tmp))
2477                 reloopBlock->append(opcode, m_value, imm(atomic->child(0)), oldValue, newValue);
2478             else if (imm(atomic->child(0)) && isValidForm(opcode, Arg::Tmp, Arg::Imm, Arg::Tmp))
2479                 reloopBlock->append(opcode, m_value, oldValue, imm(atomic->child(0)), newValue);
2480             else if (commutativity == Commutative && bitImm(atomic->child(0)) && isValidForm(opcode, Arg::BitImm, Arg::Tmp, Arg::Tmp))
2481                 reloopBlock->append(opcode, m_value, bitImm(atomic->child(0)), oldValue, newValue);
2482             else if (isValidForm(opcode, Arg::Tmp, Arg::Tmp, Arg::Tmp))
2483                 reloopBlock->append(opcode, m_value, oldValue, tmp(atomic->child(0)), newValue);
2484             else {
2485                 reloopBlock->append(relaxedMoveForType(atomic->type()), m_value, oldValue, newValue);
2486                 if (imm(atomic->child(0)) && isValidForm(opcode, Arg::Imm, Arg::Tmp))
2487                     reloopBlock->append(opcode, m_value, imm(atomic->child(0)), newValue);
2488                 else
2489                     reloopBlock->append(opcode, m_value, tmp(atomic->child(0)), newValue);
2490             }
2491         }
2492
2493         if (isX86()) {
2494             Air::Opcode casOpcode = OPCODE_FOR_WIDTH(BranchAtomicStrongCAS, atomic->accessWidth());
2495             reloopBlock->append(relaxedMoveForType(atomic->type()), m_value, oldValue, m_eax);
2496             reloopBlock->append(trappingInst(m_value, casOpcode, m_value, Arg::statusCond(MacroAssembler::Success), m_eax, newValue, address));
2497         } else {
2498             RELEASE_ASSERT(isARM64());
2499             Tmp boolResult = m_code.newTmp(GP);
2500             reloopBlock->append(trappingInst(m_value, storeCondOpcode(atomic->accessWidth(), atomic->hasFence()), m_value, newValue, address, boolResult));
2501             reloopBlock->append(BranchTest32, m_value, Arg::resCond(MacroAssembler::Zero), boolResult, boolResult);
2502         }
2503         reloopBlock->setSuccessors(doneBlock, reloopBlock);
2504     }
2505     
2506     void lower()
2507     {
2508         using namespace Air;
2509         switch (m_value->opcode()) {
2510         case B3::Nop: {
2511             // Yes, we will totally see Nop's because some phases will replaceWithNop() instead of
2512             // properly removing things.
2513             return;
2514         }
2515             
2516         case Load: {
2517             MemoryValue* memory = m_value->as<MemoryValue>();
2518             Air::Kind kind = moveForType(memory->type());
2519             if (memory->hasFence()) {
2520                 if (isX86())
2521                     kind.effects = true;
2522                 else {
2523                     switch (memory->type().kind()) {
2524                     case Int32:
2525                         kind = LoadAcq32;
2526                         break;
2527                     case Int64:
2528                         kind = LoadAcq64;
2529                         break;
2530                     default:
2531                         RELEASE_ASSERT_NOT_REACHED();
2532                         break;
2533                     }
2534                 }
2535             }
2536             append(trappingInst(m_value, kind, m_value, addr(m_value), tmp(m_value)));
2537             return;
2538         }
2539             
2540         case Load8S: {
2541             Air::Kind kind = Load8SignedExtendTo32;
2542             if (m_value->as<MemoryValue>()->hasFence()) {
2543                 if (isX86())
2544                     kind.effects = true;
2545                 else
2546                     kind = LoadAcq8SignedExtendTo32;
2547             }
2548             append(trappingInst(m_value, kind, m_value, addr(m_value), tmp(m_value)));
2549             return;
2550         }
2551
2552         case Load8Z: {
2553             Air::Kind kind = Load8;
2554             if (m_value->as<MemoryValue>()->hasFence()) {
2555                 if (isX86())
2556                     kind.effects = true;
2557                 else
2558                     kind = LoadAcq8;
2559             }
2560             append(trappingInst(m_value, kind, m_value, addr(m_value), tmp(m_value)));
2561             return;
2562         }
2563
2564         case Load16S: {
2565             Air::Kind kind = Load16SignedExtendTo32;
2566             if (m_value->as<MemoryValue>()->hasFence()) {
2567                 if (isX86())
2568                     kind.effects = true;
2569                 else
2570                     kind = LoadAcq16SignedExtendTo32;
2571             }
2572             append(trappingInst(m_value, kind, m_value, addr(m_value), tmp(m_value)));
2573             return;
2574         }
2575
2576         case Load16Z: {
2577             Air::Kind kind = Load16;
2578             if (m_value->as<MemoryValue>()->hasFence()) {
2579                 if (isX86())
2580                     kind.effects = true;
2581                 else
2582                     kind = LoadAcq16;
2583             }
2584             append(trappingInst(m_value, kind, m_value, addr(m_value), tmp(m_value)));
2585             return;
2586         }
2587             
2588         case Add: {
2589             if (tryAppendLea())
2590                 return;
2591             
2592             Air::Opcode multiplyAddOpcode = tryOpcodeForType(MultiplyAdd32, MultiplyAdd64, m_value->type());
2593             if (isValidForm(multiplyAddOpcode, Arg::Tmp, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2594                 Value* left = m_value->child(0);
2595                 Value* right = m_value->child(1);
2596                 if (!imm(right) || m_valueToTmp[right]) {
2597                     auto tryAppendMultiplyAdd = [&] (Value* left, Value* right) -> bool {
2598                         if (left->opcode() != Mul || !canBeInternal(left))
2599                             return false;
2600
2601                         Value* multiplyLeft = left->child(0);
2602                         Value* multiplyRight = left->child(1);
2603                         if (canBeInternal(multiplyLeft) || canBeInternal(multiplyRight))
2604                             return false;
2605
2606                         append(multiplyAddOpcode, tmp(multiplyLeft), tmp(multiplyRight), tmp(right), tmp(m_value));
2607                         commitInternal(left);
2608
2609                         return true;
2610                     };
2611
2612                     if (tryAppendMultiplyAdd(left, right))
2613                         return;
2614                     if (tryAppendMultiplyAdd(right, left))
2615                         return;
2616                 }
2617             }
2618
2619             appendBinOp<Add32, Add64, AddDouble, AddFloat, Commutative>(
2620                 m_value->child(0), m_value->child(1));
2621             return;
2622         }
2623
2624         case Sub: {
2625             Air::Opcode multiplySubOpcode = tryOpcodeForType(MultiplySub32, MultiplySub64, m_value->type());
2626             if (multiplySubOpcode != Air::Oops
2627                 && isValidForm(multiplySubOpcode, Arg::Tmp, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2628                 Value* left = m_value->child(0);
2629                 Value* right = m_value->child(1);
2630                 if (!imm(right) || m_valueToTmp[right]) {
2631                     auto tryAppendMultiplySub = [&] () -> bool {
2632                         if (right->opcode() != Mul || !canBeInternal(right))
2633                             return false;
2634
2635                         Value* multiplyLeft = right->child(0);
2636                         Value* multiplyRight = right->child(1);
2637                         if (m_locked.contains(multiplyLeft) || m_locked.contains(multiplyRight))
2638                             return false;
2639
2640                         append(multiplySubOpcode, tmp(multiplyLeft), tmp(multiplyRight), tmp(left), tmp(m_value));
2641                         commitInternal(right);
2642
2643                         return true;
2644                     };
2645
2646                     if (tryAppendMultiplySub())
2647                         return;
2648                 }
2649             }
2650
2651             appendBinOp<Sub32, Sub64, SubDouble, SubFloat>(m_value->child(0), m_value->child(1));
2652             return;
2653         }
2654
2655         case Neg: {
2656             Air::Opcode multiplyNegOpcode = tryOpcodeForType(MultiplyNeg32, MultiplyNeg64, m_value->type());
2657             if (multiplyNegOpcode != Air::Oops
2658                 && isValidForm(multiplyNegOpcode, Arg::Tmp, Arg::Tmp, Arg::Tmp)
2659                 && m_value->child(0)->opcode() == Mul
2660                 && canBeInternal(m_value->child(0))) {
2661                 Value* multiplyOperation = m_value->child(0);
2662                 Value* multiplyLeft = multiplyOperation->child(0);
2663                 Value* multiplyRight = multiplyOperation->child(1);
2664                 if (!m_locked.contains(multiplyLeft) && !m_locked.contains(multiplyRight)) {
2665                     append(multiplyNegOpcode, tmp(multiplyLeft), tmp(multiplyRight), tmp(m_value));
2666                     commitInternal(multiplyOperation);
2667                     return;
2668                 }
2669             }
2670
2671             appendUnOp<Neg32, Neg64, NegateDouble, NegateFloat>(m_value->child(0));
2672             return;
2673         }
2674
2675         case Mul: {
2676             if (m_value->type() == Int64
2677                 && isValidForm(MultiplySignExtend32, Arg::Tmp, Arg::Tmp, Arg::Tmp)
2678                 && m_value->child(0)->opcode() == SExt32
2679                 && !m_locked.contains(m_value->child(0))) {
2680                 Value* opLeft = m_value->child(0);
2681                 Value* left = opLeft->child(0);
2682                 Value* opRight = m_value->child(1);
2683                 Value* right = nullptr;
2684
2685                 if (opRight->opcode() == SExt32 && !m_locked.contains(opRight->child(0))) {
2686                     right = opRight->child(0);
2687                 } else if (m_value->child(1)->isRepresentableAs<int32_t>() && !m_locked.contains(m_value->child(1))) {
2688                     // We just use the 64-bit const int as a 32 bit const int directly
2689                     right = opRight;
2690                 }
2691
2692                 if (right) {
2693                     append(MultiplySignExtend32, tmp(left), tmp(right), tmp(m_value));
2694                     return;
2695                 }
2696             }
2697             appendBinOp<Mul32, Mul64, MulDouble, MulFloat, Commutative>(
2698                 m_value->child(0), m_value->child(1));
2699             return;
2700         }
2701
2702         case Div: {
2703             if (m_value->isChill())
2704                 RELEASE_ASSERT(isARM64());
2705             if (m_value->type().isInt() && isX86()) {
2706                 appendX86Div(Div);
2707                 return;
2708             }
2709             ASSERT(!isX86() || m_value->type().isFloat());
2710
2711             appendBinOp<Div32, Div64, DivDouble, DivFloat>(m_value->child(0), m_value->child(1));
2712             return;
2713         }
2714
2715         case UDiv: {
2716             if (m_value->type().isInt() && isX86()) {
2717                 appendX86UDiv(UDiv);
2718                 return;
2719             }
2720
2721             ASSERT(!isX86() && !m_value->type().isFloat());
2722
2723             appendBinOp<UDiv32, UDiv64, Air::Oops, Air::Oops>(m_value->child(0), m_value->child(1));
2724             return;
2725
2726         }
2727
2728         case Mod: {
2729             RELEASE_ASSERT(isX86());
2730             RELEASE_ASSERT(!m_value->isChill());
2731             appendX86Div(Mod);
2732             return;
2733         }
2734
2735         case UMod: {
2736             RELEASE_ASSERT(isX86());
2737             appendX86UDiv(UMod);
2738             return;
2739         }
2740
2741         case BitAnd: {
2742             if (m_value->child(1)->isInt(0xff)) {
2743                 appendUnOp<ZeroExtend8To32, ZeroExtend8To32>(m_value->child(0));
2744                 return;
2745             }
2746             
2747             if (m_value->child(1)->isInt(0xffff)) {
2748                 appendUnOp<ZeroExtend16To32, ZeroExtend16To32>(m_value->child(0));
2749                 return;
2750             }
2751
2752             if (m_value->child(1)->isInt64(0xffffffff) || m_value->child(1)->isInt32(0xffffffff)) {
2753                 appendUnOp<Move32, Move32>(m_value->child(0));
2754                 return;
2755             }
2756             
2757             appendBinOp<And32, And64, AndDouble, AndFloat, Commutative>(
2758                 m_value->child(0), m_value->child(1));
2759             return;
2760         }
2761
2762         case BitOr: {
2763             appendBinOp<Or32, Or64, OrDouble, OrFloat, Commutative>(
2764                 m_value->child(0), m_value->child(1));
2765             return;
2766         }
2767
2768         case BitXor: {
2769             // FIXME: If canBeInternal(child), we should generate this using the comparison path.
2770             // https://bugs.webkit.org/show_bug.cgi?id=152367
2771             
2772             if (m_value->child(1)->isInt(-1)) {
2773                 appendUnOp<Not32, Not64>(m_value->child(0));
2774                 return;
2775             }
2776             
2777             // This pattern is super useful on both x86 and ARM64, since the inversion of the CAS result
2778             // can be done with zero cost on x86 (just flip the set from E to NE) and it's a progression
2779             // on ARM64 (since STX returns 0 on success, so ordinarily we have to flip it).
2780             if (m_value->child(1)->isInt(1)
2781                 && m_value->child(0)->opcode() == AtomicWeakCAS
2782                 && canBeInternal(m_value->child(0))) {
2783                 commitInternal(m_value->child(0));
2784                 appendCAS(m_value->child(0), true);
2785                 return;
2786             }
2787             
2788             appendBinOp<Xor32, Xor64, XorDouble, XorFloat, Commutative>(
2789                 m_value->child(0), m_value->child(1));
2790             return;
2791         }
2792             
2793         case Depend: {
2794             RELEASE_ASSERT(isARM64());
2795             appendUnOp<Depend32, Depend64>(m_value->child(0));
2796             return;
2797         }
2798
2799         case Shl: {
2800             if (m_value->child(1)->isInt32(1)) {
2801                 appendBinOp<Add32, Add64, AddDouble, AddFloat, Commutative>(m_value->child(0), m_value->child(0));
2802                 return;
2803             }
2804             
2805             appendShift<Lshift32, Lshift64>(m_value->child(0), m_value->child(1));
2806             return;
2807         }
2808
2809         case SShr: {
2810             appendShift<Rshift32, Rshift64>(m_value->child(0), m_value->child(1));
2811             return;
2812         }
2813
2814         case ZShr: {
2815             appendShift<Urshift32, Urshift64>(m_value->child(0), m_value->child(1));
2816             return;
2817         }
2818
2819         case RotR: {
2820             appendShift<RotateRight32, RotateRight64>(m_value->child(0), m_value->child(1));
2821             return;
2822         }
2823
2824         case RotL: {
2825             appendShift<RotateLeft32, RotateLeft64>(m_value->child(0), m_value->child(1));
2826             return;
2827         }
2828
2829         case Clz: {
2830             appendUnOp<CountLeadingZeros32, CountLeadingZeros64>(m_value->child(0));
2831             return;
2832         }
2833
2834         case Abs: {
2835             RELEASE_ASSERT_WITH_MESSAGE(!isX86(), "Abs is not supported natively on x86. It must be replaced before generation.");
2836             appendUnOp<Air::Oops, Air::Oops, AbsDouble, AbsFloat>(m_value->child(0));
2837             return;
2838         }
2839
2840         case Ceil: {
2841             appendUnOp<Air::Oops, Air::Oops, CeilDouble, CeilFloat>(m_value->child(0));
2842             return;
2843         }
2844
2845         case Floor: {
2846             appendUnOp<Air::Oops, Air::Oops, FloorDouble, FloorFloat>(m_value->child(0));
2847             return;
2848         }
2849
2850         case Sqrt: {
2851             appendUnOp<Air::Oops, Air::Oops, SqrtDouble, SqrtFloat>(m_value->child(0));
2852             return;
2853         }
2854
2855         case BitwiseCast: {
2856             appendUnOp<Move32ToFloat, Move64ToDouble, MoveDoubleTo64, MoveFloatTo32>(m_value->child(0));
2857             return;
2858         }
2859
2860         case Store: {
2861             Value* valueToStore = m_value->child(0);
2862             if (canBeInternal(valueToStore)) {
2863                 bool matched = false;
2864                 switch (valueToStore->opcode()) {
2865                 case Add:
2866                     matched = tryAppendStoreBinOp<Add32, Add64, Commutative>(
2867                         valueToStore->child(0), valueToStore->child(1));
2868                     break;
2869                 case Sub:
2870                     if (valueToStore->child(0)->isInt(0)) {
2871                         matched = tryAppendStoreUnOp<Neg32, Neg64>(valueToStore->child(1));
2872                         break;
2873                     }
2874                     matched = tryAppendStoreBinOp<Sub32, Sub64>(
2875                         valueToStore->child(0), valueToStore->child(1));
2876                     break;
2877                 case BitAnd:
2878                     matched = tryAppendStoreBinOp<And32, And64, Commutative>(
2879                         valueToStore->child(0), valueToStore->child(1));
2880                     break;
2881                 case BitXor:
2882                     if (valueToStore->child(1)->isInt(-1)) {
2883                         matched = tryAppendStoreUnOp<Not32, Not64>(valueToStore->child(0));
2884                         break;
2885                     }
2886                     matched = tryAppendStoreBinOp<Xor32, Xor64, Commutative>(
2887                         valueToStore->child(0), valueToStore->child(1));
2888                     break;
2889                 default:
2890                     break;
2891                 }
2892                 if (matched) {
2893                     commitInternal(valueToStore);
2894                     return;
2895                 }
2896             }
2897
2898             appendStore(m_value, addr(m_value));
2899             return;
2900         }
2901
2902         case B3::Store8: {
2903             Value* valueToStore = m_value->child(0);
2904             if (canBeInternal(valueToStore)) {
2905                 bool matched = false;
2906                 switch (valueToStore->opcode()) {
2907                 case Add:
2908                     matched = tryAppendStoreBinOp<Add8, Air::Oops, Commutative>(
2909                         valueToStore->child(0), valueToStore->child(1));
2910                     break;
2911                 default:
2912                     break;
2913                 }
2914                 if (matched) {
2915                     commitInternal(valueToStore);
2916                     return;
2917                 }
2918             }
2919             appendStore(m_value, addr(m_value));
2920             return;
2921         }
2922
2923         case B3::Store16: {
2924             Value* valueToStore = m_value->child(0);
2925             if (canBeInternal(valueToStore)) {
2926                 bool matched = false;
2927                 switch (valueToStore->opcode()) {
2928                 case Add:
2929                     matched = tryAppendStoreBinOp<Add16, Air::Oops, Commutative>(
2930                         valueToStore->child(0), valueToStore->child(1));
2931                     break;
2932                 default:
2933                     break;
2934                 }
2935                 if (matched) {
2936                     commitInternal(valueToStore);
2937                     return;
2938                 }
2939             }
2940             appendStore(m_value, addr(m_value));
2941             return;
2942         }
2943
2944         case WasmAddress: {
2945             WasmAddressValue* address = m_value->as<WasmAddressValue>();
2946
2947             append(Add64, Arg(address->pinnedGPR()), tmp(m_value->child(0)), tmp(address));
2948             return;
2949         }
2950
2951         case Fence: {
2952             FenceValue* fence = m_value->as<FenceValue>();
2953             if (!fence->write && !fence->read)
2954                 return;
2955             if (!fence->write) {
2956                 // A fence that reads but does not write is for protecting motion of stores.
2957                 append(StoreFence);
2958                 return;
2959             }
2960             if (!fence->read) {
2961                 // A fence that writes but does not read is for protecting motion of loads.
2962                 append(LoadFence);
2963                 return;
2964             }
2965             append(MemoryFence);
2966             return;
2967         }
2968
2969         case Trunc: {
2970             ASSERT(tmp(m_value->child(0)) == tmp(m_value));
2971             return;
2972         }
2973
2974         case SExt8: {
2975             appendUnOp<SignExtend8To32, Air::Oops>(m_value->child(0));
2976             return;
2977         }
2978
2979         case SExt16: {
2980             appendUnOp<SignExtend16To32, Air::Oops>(m_value->child(0));
2981             return;
2982         }
2983
2984         case ZExt32: {
2985             appendUnOp<Move32, Air::Oops>(m_value->child(0));
2986             return;
2987         }
2988
2989         case SExt32: {
2990             // FIXME: We should have support for movsbq/movswq
2991             // https://bugs.webkit.org/show_bug.cgi?id=152232
2992             
2993             appendUnOp<SignExtend32ToPtr, Air::Oops>(m_value->child(0));
2994             return;
2995         }
2996
2997         case FloatToDouble: {
2998             appendUnOp<Air::Oops, Air::Oops, Air::Oops, ConvertFloatToDouble>(m_value->child(0));
2999             return;
3000         }
3001
3002         case DoubleToFloat: {
3003             appendUnOp<Air::Oops, Air::Oops, ConvertDoubleToFloat>(m_value->child(0));
3004             return;
3005         }
3006
3007         case ArgumentReg: {
3008             m_prologue.append(Inst(
3009                 moveForType(m_value->type()), m_value,
3010                 Tmp(m_value->as<ArgumentRegValue>()->argumentReg()),
3011                 tmp(m_value)));
3012             return;
3013         }
3014
3015         case Const32:
3016         case Const64: {
3017             if (imm(m_value))
3018                 append(Move, imm(m_value), tmp(m_value));
3019             else
3020                 append(Move, Arg::bigImm(m_value->asInt()), tmp(m_value));
3021             return;
3022         }
3023
3024         case ConstDouble:
3025         case ConstFloat: {
3026             // We expect that the moveConstants() phase has run, and any doubles referenced from
3027             // stackmaps get fused.
3028             RELEASE_ASSERT(m_value->opcode() == ConstFloat || isIdentical(m_value->asDouble(), 0.0));
3029             RELEASE_ASSERT(m_value->opcode() == ConstDouble || isIdentical(m_value->asFloat(), 0.0f));
3030             append(MoveZeroToDouble, tmp(m_value));
3031             return;
3032         }
3033
3034         case FramePointer: {
3035             ASSERT(tmp(m_value) == Tmp(GPRInfo::callFrameRegister));
3036             return;
3037         }
3038
3039         case SlotBase: {
3040             append(
3041                 pointerType() == Int64 ? Lea64 : Lea32,
3042                 Arg::stack(m_stackToStack.get(m_value->as<SlotBaseValue>()->slot())),
3043                 tmp(m_value));
3044             return;
3045         }
3046
3047         case Equal:
3048         case NotEqual: {
3049             // FIXME: Teach this to match patterns that arise from subwidth CAS. The CAS's result has to
3050             // be either zero- or sign-extended, and the value it's compared to should also be zero- or
3051             // sign-extended in a matching way. It's not super clear that this is very profitable.
3052             // https://bugs.webkit.org/show_bug.cgi?id=169250
3053             if (m_value->child(0)->opcode() == AtomicStrongCAS
3054                 && m_value->child(0)->as<AtomicValue>()->isCanonicalWidth()
3055                 && m_value->child(0)->child(0) == m_value->child(1)
3056                 && canBeInternal(m_value->child(0))) {
3057                 ASSERT(!m_locked.contains(m_value->child(0)->child(1)));
3058                 ASSERT(!m_locked.contains(m_value->child(1)));
3059                 
3060                 commitInternal(m_value->child(0));
3061                 appendCAS(m_value->child(0), m_value->opcode() == NotEqual);
3062                 return;
3063             }
3064                 
3065             m_insts.last().append(createCompare(m_value));
3066             return;
3067         }
3068             
3069         case LessThan:
3070         case GreaterThan:
3071         case LessEqual:
3072         case GreaterEqual:
3073         case Above:
3074         case Below:
3075         case AboveEqual:
3076      &