testb3 and testair should test O0/O1/O2
[WebKit-https.git] / Source / JavaScriptCore / b3 / air / testair.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #include "AirCode.h"
29 #include "AirGenerate.h"
30 #include "AirInstInlines.h"
31 #include "AirSpecial.h"
32 #include "AllowMacroScratchRegisterUsage.h"
33 #include "B3BasicBlockInlines.h"
34 #include "B3Compilation.h"
35 #include "B3Procedure.h"
36 #include "B3PatchpointSpecial.h"
37 #include "CCallHelpers.h"
38 #include "InitializeThreading.h"
39 #include "JSCInlines.h"
40 #include "LinkBuffer.h"
41 #include "PureNaN.h"
42 #include <cmath>
43 #include <string>
44 #include <wtf/Lock.h>
45 #include <wtf/NumberOfCores.h>
46 #include <wtf/StdMap.h>
47 #include <wtf/Threading.h>
48
49 // We don't have a NO_RETURN_DUE_TO_EXIT, nor should we. That's ridiculous.
50 static bool hiddenTruthBecauseNoReturnIsStupid() { return true; }
51
52 static void usage()
53 {
54     dataLog("Usage: testair [<filter>]\n");
55     if (hiddenTruthBecauseNoReturnIsStupid())
56         exit(1);
57 }
58
59 #if ENABLE(B3_JIT)
60
61 using namespace JSC;
62 using namespace JSC::B3::Air;
63
64 using JSC::B3::FP;
65 using JSC::B3::GP;
66 using JSC::B3::Width;
67 using JSC::B3::Width8;
68 using JSC::B3::Width16;
69 using JSC::B3::Width32;
70 using JSC::B3::Width64;
71
72 namespace {
73
74 Lock crashLock;
75
76 // Nothing fancy for now; we just use the existing WTF assertion machinery.
77 #define CHECK(x) do {                                                   \
78         if (!!(x))                                                      \
79             break;                                                      \
80         crashLock.lock();                                               \
81         WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #x); \
82         CRASH();                                                        \
83     } while (false)
84
85 std::unique_ptr<B3::Compilation> compile(B3::Procedure& proc)
86 {
87     prepareForGeneration(proc.code());
88     CCallHelpers jit;
89     generate(proc.code(), jit);
90     LinkBuffer linkBuffer(jit, nullptr);
91
92     return std::make_unique<B3::Compilation>(
93         FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testair compilation"), proc.releaseByproducts());
94 }
95
96 template<typename T, typename... Arguments>
97 T invoke(const B3::Compilation& code, Arguments... arguments)
98 {
99     void* executableAddress = untagCFunctionPtr(code.code().executableAddress(), B3CompilationPtrTag);
100     T (*function)(Arguments...) = bitwise_cast<T(*)(Arguments...)>(executableAddress);
101     return function(arguments...);
102 }
103
104 template<typename T, typename... Arguments>
105 T compileAndRun(B3::Procedure& procedure, Arguments... arguments)
106 {
107     return invoke<T>(*compile(procedure), arguments...);
108 }
109
110 void testSimple()
111 {
112     B3::Procedure proc;
113     Code& code = proc.code();
114
115     BasicBlock* root = code.addBlock();
116     root->append(Move, nullptr, Arg::imm(42), Tmp(GPRInfo::returnValueGPR));
117     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
118
119     CHECK(compileAndRun<int>(proc) == 42);
120 }
121
122 // Use this to put a constant into a register without Air being able to see the constant.
123 template<typename T>
124 void loadConstantImpl(BasicBlock* block, T value, B3::Air::Opcode move, Tmp tmp, Tmp scratch)
125 {
126     static Lock lock;
127     static StdMap<T, T*>* map; // I'm not messing with HashMap's problems with integers.
128
129     LockHolder locker(lock);
130     if (!map)
131         map = new StdMap<T, T*>();
132
133     if (!map->count(value))
134         (*map)[value] = new T(value);
135
136     T* ptr = (*map)[value];
137     block->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(ptr)), scratch);
138     block->append(move, nullptr, Arg::addr(scratch), tmp);
139 }
140
141 template<typename T>
142 void loadConstant(BasicBlock* block, T value, Tmp tmp)
143 {
144     loadConstantImpl(block, value, Move, tmp, tmp);
145 }
146
147 void loadDoubleConstant(BasicBlock* block, double value, Tmp tmp, Tmp scratch)
148 {
149     loadConstantImpl<double>(block, value, MoveDouble, tmp, scratch);
150 }
151
152 void testShuffleSimpleSwap()
153 {
154     B3::Procedure proc;
155     Code& code = proc.code();
156
157     BasicBlock* root = code.addBlock();
158     loadConstant(root, 1, Tmp(GPRInfo::regT0));
159     loadConstant(root, 2, Tmp(GPRInfo::regT1));
160     loadConstant(root, 3, Tmp(GPRInfo::regT2));
161     loadConstant(root, 4, Tmp(GPRInfo::regT3));
162     root->append(
163         Shuffle, nullptr,
164         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
165         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width32));
166
167     int32_t things[4];
168     Tmp base = code.newTmp(GP);
169     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
170     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
171     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
172     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
173     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
174     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
175     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
176
177     memset(things, 0, sizeof(things));
178     
179     CHECK(!compileAndRun<int>(proc));
180
181     CHECK(things[0] == 1);
182     CHECK(things[1] == 2);
183     CHECK(things[2] == 4);
184     CHECK(things[3] == 3);
185 }
186
187 void testShuffleSimpleShift()
188 {
189     B3::Procedure proc;
190     Code& code = proc.code();
191
192     BasicBlock* root = code.addBlock();
193     loadConstant(root, 1, Tmp(GPRInfo::regT0));
194     loadConstant(root, 2, Tmp(GPRInfo::regT1));
195     loadConstant(root, 3, Tmp(GPRInfo::regT2));
196     loadConstant(root, 4, Tmp(GPRInfo::regT3));
197     root->append(
198         Shuffle, nullptr,
199         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
200         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32));
201
202     int32_t things[5];
203     Tmp base = code.newTmp(GP);
204     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
205     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
206     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
207     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
208     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
209     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
210     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
211     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
212
213     memset(things, 0, sizeof(things));
214     
215     CHECK(!compileAndRun<int>(proc));
216
217     CHECK(things[0] == 1);
218     CHECK(things[1] == 2);
219     CHECK(things[2] == 3);
220     CHECK(things[3] == 3);
221     CHECK(things[4] == 4);
222 }
223
224 void testShuffleLongShift()
225 {
226     B3::Procedure proc;
227     Code& code = proc.code();
228
229     BasicBlock* root = code.addBlock();
230     loadConstant(root, 1, Tmp(GPRInfo::regT0));
231     loadConstant(root, 2, Tmp(GPRInfo::regT1));
232     loadConstant(root, 3, Tmp(GPRInfo::regT2));
233     loadConstant(root, 4, Tmp(GPRInfo::regT3));
234     loadConstant(root, 5, Tmp(GPRInfo::regT4));
235     loadConstant(root, 6, Tmp(GPRInfo::regT5));
236     loadConstant(root, 7, Tmp(GPRInfo::regT6));
237     loadConstant(root, 8, Tmp(GPRInfo::regT7));
238     root->append(
239         Shuffle, nullptr,
240         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
241         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
242         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
243         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
244         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
245         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
246         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT7), Arg::widthArg(Width32));
247
248     int32_t things[8];
249     Tmp base = code.newTmp(GP);
250     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
251     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
252     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
253     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
254     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
255     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
256     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
257     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
258     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
259     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
260     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
261
262     memset(things, 0, sizeof(things));
263     
264     CHECK(!compileAndRun<int>(proc));
265
266     CHECK(things[0] == 1);
267     CHECK(things[1] == 1);
268     CHECK(things[2] == 2);
269     CHECK(things[3] == 3);
270     CHECK(things[4] == 4);
271     CHECK(things[5] == 5);
272     CHECK(things[6] == 6);
273     CHECK(things[7] == 7);
274 }
275
276 void testShuffleLongShiftBackwards()
277 {
278     B3::Procedure proc;
279     Code& code = proc.code();
280
281     BasicBlock* root = code.addBlock();
282     loadConstant(root, 1, Tmp(GPRInfo::regT0));
283     loadConstant(root, 2, Tmp(GPRInfo::regT1));
284     loadConstant(root, 3, Tmp(GPRInfo::regT2));
285     loadConstant(root, 4, Tmp(GPRInfo::regT3));
286     loadConstant(root, 5, Tmp(GPRInfo::regT4));
287     loadConstant(root, 6, Tmp(GPRInfo::regT5));
288     loadConstant(root, 7, Tmp(GPRInfo::regT6));
289     loadConstant(root, 8, Tmp(GPRInfo::regT7));
290     root->append(
291         Shuffle, nullptr,
292         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
293         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
294         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
295         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
296         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
297         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
298         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
299
300     int32_t things[8];
301     Tmp base = code.newTmp(GP);
302     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
303     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
304     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
305     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
306     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
307     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
308     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
309     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
310     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
311     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
312     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
313
314     memset(things, 0, sizeof(things));
315     
316     CHECK(!compileAndRun<int>(proc));
317
318     CHECK(things[0] == 1);
319     CHECK(things[1] == 1);
320     CHECK(things[2] == 2);
321     CHECK(things[3] == 3);
322     CHECK(things[4] == 4);
323     CHECK(things[5] == 5);
324     CHECK(things[6] == 6);
325     CHECK(things[7] == 7);
326 }
327
328 void testShuffleSimpleRotate()
329 {
330     B3::Procedure proc;
331     Code& code = proc.code();
332
333     BasicBlock* root = code.addBlock();
334     loadConstant(root, 1, Tmp(GPRInfo::regT0));
335     loadConstant(root, 2, Tmp(GPRInfo::regT1));
336     loadConstant(root, 3, Tmp(GPRInfo::regT2));
337     loadConstant(root, 4, Tmp(GPRInfo::regT3));
338     root->append(
339         Shuffle, nullptr,
340         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
341         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
342         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32));
343
344     int32_t things[4];
345     Tmp base = code.newTmp(GP);
346     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
347     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
348     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
349     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
350     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
351     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
352     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
353
354     memset(things, 0, sizeof(things));
355     
356     CHECK(!compileAndRun<int>(proc));
357
358     CHECK(things[0] == 3);
359     CHECK(things[1] == 1);
360     CHECK(things[2] == 2);
361     CHECK(things[3] == 4);
362 }
363
364 void testShuffleSimpleBroadcast()
365 {
366     B3::Procedure proc;
367     Code& code = proc.code();
368
369     BasicBlock* root = code.addBlock();
370     loadConstant(root, 1, Tmp(GPRInfo::regT0));
371     loadConstant(root, 2, Tmp(GPRInfo::regT1));
372     loadConstant(root, 3, Tmp(GPRInfo::regT2));
373     loadConstant(root, 4, Tmp(GPRInfo::regT3));
374     root->append(
375         Shuffle, nullptr,
376         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
377         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
378         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32));
379
380     int32_t things[4];
381     Tmp base = code.newTmp(GP);
382     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
383     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
384     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
385     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
386     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
387     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
388     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
389
390     memset(things, 0, sizeof(things));
391     
392     CHECK(!compileAndRun<int>(proc));
393
394     CHECK(things[0] == 1);
395     CHECK(things[1] == 1);
396     CHECK(things[2] == 1);
397     CHECK(things[3] == 1);
398 }
399
400 void testShuffleBroadcastAllRegs()
401 {
402     B3::Procedure proc;
403     Code& code = proc.code();
404
405     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
406
407     BasicBlock* root = code.addBlock();
408     root->append(Move, nullptr, Arg::imm(35), Tmp(GPRInfo::regT0));
409     unsigned count = 1;
410     for (Reg reg : regs) {
411         if (reg != Reg(GPRInfo::regT0))
412             loadConstant(root, count++, Tmp(reg));
413     }
414     Inst& shuffle = root->append(Shuffle, nullptr);
415     for (Reg reg : regs) {
416         if (reg != Reg(GPRInfo::regT0))
417             shuffle.append(Tmp(GPRInfo::regT0), Tmp(reg), Arg::widthArg(Width32));
418     }
419
420     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
421     for (unsigned i = 0; i < regs.size(); ++i)
422         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
423
424     Vector<int32_t> things(regs.size(), 666);
425     Tmp base = code.newTmp(GP);
426     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
427     for (unsigned i = 0; i < regs.size(); ++i) {
428         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
429         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
430     }
431     
432     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
433     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
434
435     CHECK(!compileAndRun<int>(proc));
436
437     for (int32_t thing : things)
438         CHECK(thing == 35);
439 }
440
441 void testShuffleTreeShift()
442 {
443     B3::Procedure proc;
444     Code& code = proc.code();
445
446     BasicBlock* root = code.addBlock();
447     loadConstant(root, 1, Tmp(GPRInfo::regT0));
448     loadConstant(root, 2, Tmp(GPRInfo::regT1));
449     loadConstant(root, 3, Tmp(GPRInfo::regT2));
450     loadConstant(root, 4, Tmp(GPRInfo::regT3));
451     loadConstant(root, 5, Tmp(GPRInfo::regT4));
452     loadConstant(root, 6, Tmp(GPRInfo::regT5));
453     loadConstant(root, 7, Tmp(GPRInfo::regT6));
454     loadConstant(root, 8, Tmp(GPRInfo::regT7));
455     root->append(
456         Shuffle, nullptr,
457         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
458         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
459         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
460         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
461         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
462         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
463         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT7), Arg::widthArg(Width32));
464
465     int32_t things[8];
466     Tmp base = code.newTmp(GP);
467     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
468     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
469     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
470     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
471     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
472     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
473     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
474     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
475     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
476     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
477     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
478
479     memset(things, 0, sizeof(things));
480     
481     CHECK(!compileAndRun<int>(proc));
482
483     CHECK(things[0] == 1);
484     CHECK(things[1] == 1);
485     CHECK(things[2] == 1);
486     CHECK(things[3] == 2);
487     CHECK(things[4] == 2);
488     CHECK(things[5] == 3);
489     CHECK(things[6] == 3);
490     CHECK(things[7] == 4);
491 }
492
493 void testShuffleTreeShiftBackward()
494 {
495     B3::Procedure proc;
496     Code& code = proc.code();
497
498     BasicBlock* root = code.addBlock();
499     loadConstant(root, 1, Tmp(GPRInfo::regT0));
500     loadConstant(root, 2, Tmp(GPRInfo::regT1));
501     loadConstant(root, 3, Tmp(GPRInfo::regT2));
502     loadConstant(root, 4, Tmp(GPRInfo::regT3));
503     loadConstant(root, 5, Tmp(GPRInfo::regT4));
504     loadConstant(root, 6, Tmp(GPRInfo::regT5));
505     loadConstant(root, 7, Tmp(GPRInfo::regT6));
506     loadConstant(root, 8, Tmp(GPRInfo::regT7));
507     root->append(
508         Shuffle, nullptr,
509         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
510         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
511         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
512         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
513         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
514         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
515         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
516
517     int32_t things[8];
518     Tmp base = code.newTmp(GP);
519     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
520     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
521     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
522     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
523     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
524     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
525     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
526     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
527     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
528     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
529     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
530
531     memset(things, 0, sizeof(things));
532     
533     CHECK(!compileAndRun<int>(proc));
534
535     CHECK(things[0] == 1);
536     CHECK(things[1] == 1);
537     CHECK(things[2] == 1);
538     CHECK(things[3] == 2);
539     CHECK(things[4] == 2);
540     CHECK(things[5] == 3);
541     CHECK(things[6] == 3);
542     CHECK(things[7] == 4);
543 }
544
545 void testShuffleTreeShiftOtherBackward()
546 {
547     // NOTE: This test was my original attempt at TreeShiftBackward but mistakes were made. So, this
548     // ends up being just a weird test. But weird tests are useful, so I kept it.
549     
550     B3::Procedure proc;
551     Code& code = proc.code();
552
553     BasicBlock* root = code.addBlock();
554     loadConstant(root, 1, Tmp(GPRInfo::regT0));
555     loadConstant(root, 2, Tmp(GPRInfo::regT1));
556     loadConstant(root, 3, Tmp(GPRInfo::regT2));
557     loadConstant(root, 4, Tmp(GPRInfo::regT3));
558     loadConstant(root, 5, Tmp(GPRInfo::regT4));
559     loadConstant(root, 6, Tmp(GPRInfo::regT5));
560     loadConstant(root, 7, Tmp(GPRInfo::regT6));
561     loadConstant(root, 8, Tmp(GPRInfo::regT7));
562     root->append(
563         Shuffle, nullptr,
564         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
565         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
566         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
567         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
568         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
569         Tmp(GPRInfo::regT7), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
570         Tmp(GPRInfo::regT7), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
571
572     int32_t things[8];
573     Tmp base = code.newTmp(GP);
574     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
575     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
576     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
577     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
578     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
579     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
580     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
581     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
582     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
583     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
584     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
585
586     memset(things, 0, sizeof(things));
587     
588     CHECK(!compileAndRun<int>(proc));
589
590     CHECK(things[0] == 1);
591     CHECK(things[1] == 8);
592     CHECK(things[2] == 8);
593     CHECK(things[3] == 7);
594     CHECK(things[4] == 7);
595     CHECK(things[5] == 6);
596     CHECK(things[6] == 6);
597     CHECK(things[7] == 5);
598 }
599
600 void testShuffleMultipleShifts()
601 {
602     B3::Procedure proc;
603     Code& code = proc.code();
604
605     BasicBlock* root = code.addBlock();
606     loadConstant(root, 1, Tmp(GPRInfo::regT0));
607     loadConstant(root, 2, Tmp(GPRInfo::regT1));
608     loadConstant(root, 3, Tmp(GPRInfo::regT2));
609     loadConstant(root, 4, Tmp(GPRInfo::regT3));
610     loadConstant(root, 5, Tmp(GPRInfo::regT4));
611     loadConstant(root, 6, Tmp(GPRInfo::regT5));
612     root->append(
613         Shuffle, nullptr,
614         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
615         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
616         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
617         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
618
619     int32_t things[6];
620     Tmp base = code.newTmp(GP);
621     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
622     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
623     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
624     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
625     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
626     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
627     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
628     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
629     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
630
631     memset(things, 0, sizeof(things));
632     
633     CHECK(!compileAndRun<int>(proc));
634
635     CHECK(things[0] == 1);
636     CHECK(things[1] == 1);
637     CHECK(things[2] == 3);
638     CHECK(things[3] == 3);
639     CHECK(things[4] == 3);
640     CHECK(things[5] == 1);
641 }
642
643 void testShuffleRotateWithFringe()
644 {
645     B3::Procedure proc;
646     Code& code = proc.code();
647
648     BasicBlock* root = code.addBlock();
649     loadConstant(root, 1, Tmp(GPRInfo::regT0));
650     loadConstant(root, 2, Tmp(GPRInfo::regT1));
651     loadConstant(root, 3, Tmp(GPRInfo::regT2));
652     loadConstant(root, 4, Tmp(GPRInfo::regT3));
653     loadConstant(root, 5, Tmp(GPRInfo::regT4));
654     loadConstant(root, 6, Tmp(GPRInfo::regT5));
655     root->append(
656         Shuffle, nullptr,
657         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
658         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
659         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
660         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
661         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
662         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
663
664     int32_t things[6];
665     Tmp base = code.newTmp(GP);
666     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
667     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
668     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
669     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
670     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
671     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
672     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
673     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
674     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
675
676     memset(things, 0, sizeof(things));
677     
678     CHECK(!compileAndRun<int>(proc));
679
680     CHECK(things[0] == 3);
681     CHECK(things[1] == 1);
682     CHECK(things[2] == 2);
683     CHECK(things[3] == 1);
684     CHECK(things[4] == 2);
685     CHECK(things[5] == 3);
686 }
687
688 void testShuffleRotateWithFringeInWeirdOrder()
689 {
690     B3::Procedure proc;
691     Code& code = proc.code();
692
693     BasicBlock* root = code.addBlock();
694     loadConstant(root, 1, Tmp(GPRInfo::regT0));
695     loadConstant(root, 2, Tmp(GPRInfo::regT1));
696     loadConstant(root, 3, Tmp(GPRInfo::regT2));
697     loadConstant(root, 4, Tmp(GPRInfo::regT3));
698     loadConstant(root, 5, Tmp(GPRInfo::regT4));
699     loadConstant(root, 6, Tmp(GPRInfo::regT5));
700     root->append(
701         Shuffle, nullptr,
702         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
703         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
704         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
705         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
706         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
707         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32));
708
709     int32_t things[6];
710     Tmp base = code.newTmp(GP);
711     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
712     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
713     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
714     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
715     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
716     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
717     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
718     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
719     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
720
721     memset(things, 0, sizeof(things));
722     
723     CHECK(!compileAndRun<int>(proc));
724
725     CHECK(things[0] == 3);
726     CHECK(things[1] == 1);
727     CHECK(things[2] == 2);
728     CHECK(things[3] == 1);
729     CHECK(things[4] == 2);
730     CHECK(things[5] == 3);
731 }
732
733 void testShuffleRotateWithLongFringe()
734 {
735     B3::Procedure proc;
736     Code& code = proc.code();
737
738     BasicBlock* root = code.addBlock();
739     loadConstant(root, 1, Tmp(GPRInfo::regT0));
740     loadConstant(root, 2, Tmp(GPRInfo::regT1));
741     loadConstant(root, 3, Tmp(GPRInfo::regT2));
742     loadConstant(root, 4, Tmp(GPRInfo::regT3));
743     loadConstant(root, 5, Tmp(GPRInfo::regT4));
744     loadConstant(root, 6, Tmp(GPRInfo::regT5));
745     root->append(
746         Shuffle, nullptr,
747         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
748         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
749         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
750         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
751         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
752         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
753
754     int32_t things[6];
755     Tmp base = code.newTmp(GP);
756     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
757     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
758     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
759     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
760     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
761     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
762     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
763     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
764     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
765
766     memset(things, 0, sizeof(things));
767     
768     CHECK(!compileAndRun<int>(proc));
769
770     CHECK(things[0] == 3);
771     CHECK(things[1] == 1);
772     CHECK(things[2] == 2);
773     CHECK(things[3] == 1);
774     CHECK(things[4] == 4);
775     CHECK(things[5] == 5);
776 }
777
778 void testShuffleMultipleRotates()
779 {
780     B3::Procedure proc;
781     Code& code = proc.code();
782
783     BasicBlock* root = code.addBlock();
784     loadConstant(root, 1, Tmp(GPRInfo::regT0));
785     loadConstant(root, 2, Tmp(GPRInfo::regT1));
786     loadConstant(root, 3, Tmp(GPRInfo::regT2));
787     loadConstant(root, 4, Tmp(GPRInfo::regT3));
788     loadConstant(root, 5, Tmp(GPRInfo::regT4));
789     loadConstant(root, 6, Tmp(GPRInfo::regT5));
790     root->append(
791         Shuffle, nullptr,
792         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
793         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
794         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
795         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
796         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
797         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT3), Arg::widthArg(Width32));
798
799     int32_t things[6];
800     Tmp base = code.newTmp(GP);
801     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
802     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
803     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
804     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
805     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
806     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
807     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
808     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
809     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
810
811     memset(things, 0, sizeof(things));
812     
813     CHECK(!compileAndRun<int>(proc));
814
815     CHECK(things[0] == 3);
816     CHECK(things[1] == 1);
817     CHECK(things[2] == 2);
818     CHECK(things[3] == 6);
819     CHECK(things[4] == 4);
820     CHECK(things[5] == 5);
821 }
822
823 void testShuffleShiftAndRotate()
824 {
825     B3::Procedure proc;
826     Code& code = proc.code();
827
828     BasicBlock* root = code.addBlock();
829     loadConstant(root, 1, Tmp(GPRInfo::regT0));
830     loadConstant(root, 2, Tmp(GPRInfo::regT1));
831     loadConstant(root, 3, Tmp(GPRInfo::regT2));
832     loadConstant(root, 4, Tmp(GPRInfo::regT3));
833     loadConstant(root, 5, Tmp(GPRInfo::regT4));
834     loadConstant(root, 6, Tmp(GPRInfo::regT5));
835     root->append(
836         Shuffle, nullptr,
837         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
838         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
839         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
840         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
841         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
842
843     int32_t things[6];
844     Tmp base = code.newTmp(GP);
845     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
846     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
847     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
848     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
849     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
850     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
851     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
852     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
853     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
854
855     memset(things, 0, sizeof(things));
856     
857     CHECK(!compileAndRun<int>(proc));
858
859     CHECK(things[0] == 3);
860     CHECK(things[1] == 1);
861     CHECK(things[2] == 2);
862     CHECK(things[3] == 4);
863     CHECK(things[4] == 4);
864     CHECK(things[5] == 5);
865 }
866
867 void testShuffleShiftAllRegs()
868 {
869     B3::Procedure proc;
870     Code& code = proc.code();
871
872     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
873
874     BasicBlock* root = code.addBlock();
875     for (unsigned i = 0; i < regs.size(); ++i)
876         loadConstant(root, 35 + i, Tmp(regs[i]));
877     Inst& shuffle = root->append(Shuffle, nullptr);
878     for (unsigned i = 1; i < regs.size(); ++i)
879         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
880
881     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
882     for (unsigned i = 0; i < regs.size(); ++i)
883         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
884
885     Vector<int32_t> things(regs.size(), 666);
886     Tmp base = code.newTmp(GP);
887     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
888     for (unsigned i = 0; i < regs.size(); ++i) {
889         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
890         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
891     }
892     
893     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
894     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
895
896     CHECK(!compileAndRun<int>(proc));
897
898     CHECK(things[0] == 35);
899     for (unsigned i = 1; i < regs.size(); ++i)
900         CHECK(things[i] == 35 + static_cast<int32_t>(i) - 1);
901 }
902
903 void testShuffleRotateAllRegs()
904 {
905     B3::Procedure proc;
906     Code& code = proc.code();
907
908     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
909
910     BasicBlock* root = code.addBlock();
911     for (unsigned i = 0; i < regs.size(); ++i)
912         loadConstant(root, 35 + i, Tmp(regs[i]));
913     Inst& shuffle = root->append(Shuffle, nullptr);
914     for (unsigned i = 1; i < regs.size(); ++i)
915         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
916     shuffle.append(Tmp(regs.last()), Tmp(regs[0]), Arg::widthArg(Width32));
917
918     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
919     for (unsigned i = 0; i < regs.size(); ++i)
920         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
921
922     Vector<int32_t> things(regs.size(), 666);
923     Tmp base = code.newTmp(GP);
924     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
925     for (unsigned i = 0; i < regs.size(); ++i) {
926         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
927         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
928     }
929     
930     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
931     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
932
933     CHECK(!compileAndRun<int>(proc));
934
935     CHECK(things[0] == 35 + static_cast<int32_t>(regs.size()) - 1);
936     for (unsigned i = 1; i < regs.size(); ++i)
937         CHECK(things[i] == 35 + static_cast<int32_t>(i) - 1);
938 }
939
940 void testShuffleSimpleSwap64()
941 {
942     B3::Procedure proc;
943     Code& code = proc.code();
944
945     BasicBlock* root = code.addBlock();
946     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
947     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
948     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
949     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
950     root->append(
951         Shuffle, nullptr,
952         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
953         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width64));
954
955     int64_t things[4];
956     Tmp base = code.newTmp(GP);
957     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
958     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
959     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
960     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
961     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
962     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
963     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
964
965     memset(things, 0, sizeof(things));
966     
967     CHECK(!compileAndRun<int>(proc));
968
969     CHECK(things[0] == 10000000000000000ll);
970     CHECK(things[1] == 20000000000000000ll);
971     CHECK(things[2] == 40000000000000000ll);
972     CHECK(things[3] == 30000000000000000ll);
973 }
974
975 void testShuffleSimpleShift64()
976 {
977     B3::Procedure proc;
978     Code& code = proc.code();
979
980     BasicBlock* root = code.addBlock();
981     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
982     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
983     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
984     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
985     loadConstant(root, 50000000000000000ll, Tmp(GPRInfo::regT4));
986     root->append(
987         Shuffle, nullptr,
988         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
989         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width64));
990
991     int64_t things[5];
992     Tmp base = code.newTmp(GP);
993     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
994     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
995     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
996     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
997     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
998     root->append(Move, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int64_t)));
999     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1000     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1001
1002     memset(things, 0, sizeof(things));
1003     
1004     CHECK(!compileAndRun<int>(proc));
1005
1006     CHECK(things[0] == 10000000000000000ll);
1007     CHECK(things[1] == 20000000000000000ll);
1008     CHECK(things[2] == 30000000000000000ll);
1009     CHECK(things[3] == 30000000000000000ll);
1010     CHECK(things[4] == 40000000000000000ll);
1011 }
1012
1013 void testShuffleSwapMixedWidth()
1014 {
1015     B3::Procedure proc;
1016     Code& code = proc.code();
1017
1018     BasicBlock* root = code.addBlock();
1019     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
1020     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
1021     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
1022     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
1023     root->append(
1024         Shuffle, nullptr,
1025         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
1026         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width64));
1027
1028     int64_t things[4];
1029     Tmp base = code.newTmp(GP);
1030     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1031     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1032     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1033     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
1034     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
1035     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1036     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1037
1038     memset(things, 0, sizeof(things));
1039     
1040     CHECK(!compileAndRun<int>(proc));
1041
1042     CHECK(things[0] == 10000000000000000ll);
1043     CHECK(things[1] == 20000000000000000ll);
1044     CHECK(things[2] == 40000000000000000ll);
1045     CHECK(things[3] == static_cast<uint32_t>(30000000000000000ll));
1046 }
1047
1048 void testShuffleShiftMixedWidth()
1049 {
1050     B3::Procedure proc;
1051     Code& code = proc.code();
1052
1053     BasicBlock* root = code.addBlock();
1054     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
1055     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
1056     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
1057     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
1058     loadConstant(root, 50000000000000000ll, Tmp(GPRInfo::regT4));
1059     root->append(
1060         Shuffle, nullptr,
1061         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
1062         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32));
1063
1064     int64_t things[5];
1065     Tmp base = code.newTmp(GP);
1066     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1067     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1068     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1069     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
1070     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
1071     root->append(Move, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int64_t)));
1072     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1073     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1074
1075     memset(things, 0, sizeof(things));
1076     
1077     CHECK(!compileAndRun<int>(proc));
1078
1079     CHECK(things[0] == 10000000000000000ll);
1080     CHECK(things[1] == 20000000000000000ll);
1081     CHECK(things[2] == 30000000000000000ll);
1082     CHECK(things[3] == 30000000000000000ll);
1083     CHECK(things[4] == static_cast<uint32_t>(40000000000000000ll));
1084 }
1085
1086 void testShuffleShiftMemory()
1087 {
1088     B3::Procedure proc;
1089     Code& code = proc.code();
1090
1091     int32_t memory[2];
1092     memory[0] = 35;
1093     memory[1] = 36;
1094
1095     BasicBlock* root = code.addBlock();
1096     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1097     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1098     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1099     root->append(
1100         Shuffle, nullptr,
1101         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1102         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1103         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Arg::widthArg(Width32));
1104
1105     int32_t things[2];
1106     Tmp base = code.newTmp(GP);
1107     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1108     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1109     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1110     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1111     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1112
1113     memset(things, 0, sizeof(things));
1114     
1115     CHECK(!compileAndRun<int>(proc));
1116
1117     CHECK(things[0] == 1);
1118     CHECK(things[1] == 1);
1119     CHECK(memory[0] == 35);
1120     CHECK(memory[1] == 35);
1121 }
1122
1123 void testShuffleShiftMemoryLong()
1124 {
1125     B3::Procedure proc;
1126     Code& code = proc.code();
1127
1128     int32_t memory[2];
1129     memory[0] = 35;
1130     memory[1] = 36;
1131
1132     BasicBlock* root = code.addBlock();
1133     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1134     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1135     loadConstant(root, 3, Tmp(GPRInfo::regT2));
1136     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT3));
1137     root->append(
1138         Shuffle, nullptr,
1139         
1140         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1141         
1142         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT3), 0 * sizeof(int32_t)),
1143         Arg::widthArg(Width32),
1144         
1145         Arg::addr(Tmp(GPRInfo::regT3), 0 * sizeof(int32_t)),
1146         Arg::addr(Tmp(GPRInfo::regT3), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1147
1148         Arg::addr(Tmp(GPRInfo::regT3), 1 * sizeof(int32_t)), Tmp(GPRInfo::regT2),
1149         Arg::widthArg(Width32));
1150
1151     int32_t things[3];
1152     Tmp base = code.newTmp(GP);
1153     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1154     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1155     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1156     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
1157     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1158     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1159
1160     memset(things, 0, sizeof(things));
1161     
1162     CHECK(!compileAndRun<int>(proc));
1163
1164     CHECK(things[0] == 1);
1165     CHECK(things[1] == 1);
1166     CHECK(things[2] == 36);
1167     CHECK(memory[0] == 2);
1168     CHECK(memory[1] == 35);
1169 }
1170
1171 void testShuffleShiftMemoryAllRegs()
1172 {
1173     B3::Procedure proc;
1174     Code& code = proc.code();
1175
1176     int32_t memory[2];
1177     memory[0] = 35;
1178     memory[1] = 36;
1179
1180     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1181     regs.removeFirst(Reg(GPRInfo::regT0));
1182
1183     BasicBlock* root = code.addBlock();
1184     for (unsigned i = 0; i < regs.size(); ++i)
1185         loadConstant(root, i + 1, Tmp(regs[i]));
1186     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1187     Inst& shuffle = root->append(
1188         Shuffle, nullptr,
1189         
1190         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int32_t)),
1191         Arg::widthArg(Width32),
1192         
1193         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int32_t)),
1194         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1195
1196         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int32_t)), Tmp(regs[1]),
1197         Arg::widthArg(Width32));
1198
1199     for (unsigned i = 2; i < regs.size(); ++i)
1200         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
1201
1202     Vector<int32_t> things(regs.size(), 666);
1203     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1204     for (unsigned i = 0; i < regs.size(); ++i) {
1205         root->append(
1206             Move32, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int32_t)));
1207     }
1208     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1209     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1210
1211     CHECK(!compileAndRun<int>(proc));
1212
1213     CHECK(things[0] == 1);
1214     CHECK(things[1] == 36);
1215     for (unsigned i = 2; i < regs.size(); ++i)
1216         CHECK(things[i] == static_cast<int32_t>(i));
1217     CHECK(memory[0] == 1);
1218     CHECK(memory[1] == 35);
1219 }
1220
1221 void testShuffleShiftMemoryAllRegs64()
1222 {
1223     B3::Procedure proc;
1224     Code& code = proc.code();
1225
1226     int64_t memory[2];
1227     memory[0] = 35000000000000ll;
1228     memory[1] = 36000000000000ll;
1229
1230     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1231     regs.removeFirst(Reg(GPRInfo::regT0));
1232
1233     BasicBlock* root = code.addBlock();
1234     for (unsigned i = 0; i < regs.size(); ++i)
1235         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1236     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1237     Inst& shuffle = root->append(
1238         Shuffle, nullptr,
1239         
1240         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1241         Arg::widthArg(Width64),
1242         
1243         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1244         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1245
1246         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1247         Arg::widthArg(Width64));
1248
1249     for (unsigned i = 2; i < regs.size(); ++i)
1250         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1251
1252     Vector<int64_t> things(regs.size(), 666);
1253     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1254     for (unsigned i = 0; i < regs.size(); ++i) {
1255         root->append(
1256             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1257     }
1258     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1259     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1260
1261     CHECK(!compileAndRun<int>(proc));
1262
1263     CHECK(things[0] == 1000000000000ll);
1264     CHECK(things[1] == 36000000000000ll);
1265     for (unsigned i = 2; i < regs.size(); ++i)
1266         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1267     CHECK(memory[0] == 1000000000000ll);
1268     CHECK(memory[1] == 35000000000000ll);
1269 }
1270
1271 int64_t combineHiLo(int64_t high, int64_t low)
1272 {
1273     union {
1274         int64_t value;
1275         int32_t halves[2];
1276     } u;
1277     u.value = high;
1278     u.halves[0] = static_cast<int32_t>(low);
1279     return u.value;
1280 }
1281
1282 void testShuffleShiftMemoryAllRegsMixedWidth()
1283 {
1284     B3::Procedure proc;
1285     Code& code = proc.code();
1286
1287     int64_t memory[2];
1288     memory[0] = 35000000000000ll;
1289     memory[1] = 36000000000000ll;
1290
1291     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1292     regs.removeFirst(Reg(GPRInfo::regT0));
1293
1294     BasicBlock* root = code.addBlock();
1295     for (unsigned i = 0; i < regs.size(); ++i)
1296         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1297     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1298     Inst& shuffle = root->append(
1299         Shuffle, nullptr,
1300         
1301         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1302         Arg::widthArg(Width32),
1303         
1304         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1305         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1306
1307         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1308         Arg::widthArg(Width32));
1309
1310     for (unsigned i = 2; i < regs.size(); ++i) {
1311         shuffle.append(
1312             Tmp(regs[i - 1]), Tmp(regs[i]),
1313             (i & 1) ? Arg::widthArg(Width32) : Arg::widthArg(Width64));
1314     }
1315
1316     Vector<int64_t> things(regs.size(), 666);
1317     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1318     for (unsigned i = 0; i < regs.size(); ++i) {
1319         root->append(
1320             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1321     }
1322     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1323     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1324
1325     CHECK(!compileAndRun<int>(proc));
1326
1327     CHECK(things[0] == 1000000000000ll);
1328     CHECK(things[1] == static_cast<uint32_t>(36000000000000ll));
1329     for (unsigned i = 2; i < regs.size(); ++i) {
1330         int64_t value = static_cast<int64_t>(i) * 1000000000000ll;
1331         CHECK(things[i] == ((i & 1) ? static_cast<uint32_t>(value) : value));
1332     }
1333     CHECK(memory[0] == combineHiLo(35000000000000ll, 1000000000000ll));
1334     CHECK(memory[1] == 35000000000000ll);
1335 }
1336
1337 void testShuffleRotateMemory()
1338 {
1339     B3::Procedure proc;
1340     Code& code = proc.code();
1341
1342     int32_t memory[2];
1343     memory[0] = 35;
1344     memory[1] = 36;
1345
1346     BasicBlock* root = code.addBlock();
1347     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1348     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1349     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1350     root->append(
1351         Shuffle, nullptr,
1352         
1353         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1354
1355         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1356         Arg::widthArg(Width32),
1357         
1358         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1359         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1360
1361         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Tmp(GPRInfo::regT0),
1362         Arg::widthArg(Width32));
1363
1364     int32_t things[2];
1365     Tmp base = code.newTmp(GP);
1366     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1367     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1368     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1369     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1370     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1371
1372     memset(things, 0, sizeof(things));
1373     
1374     CHECK(!compileAndRun<int>(proc));
1375
1376     CHECK(things[0] == 36);
1377     CHECK(things[1] == 1);
1378     CHECK(memory[0] == 2);
1379     CHECK(memory[1] == 35);
1380 }
1381
1382 void testShuffleRotateMemory64()
1383 {
1384     B3::Procedure proc;
1385     Code& code = proc.code();
1386
1387     int64_t memory[2];
1388     memory[0] = 35000000000000ll;
1389     memory[1] = 36000000000000ll;
1390
1391     BasicBlock* root = code.addBlock();
1392     loadConstant(root, 1000000000000ll, Tmp(GPRInfo::regT0));
1393     loadConstant(root, 2000000000000ll, Tmp(GPRInfo::regT1));
1394     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1395     root->append(
1396         Shuffle, nullptr,
1397         
1398         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width64),
1399
1400         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1401         Arg::widthArg(Width64),
1402         
1403         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1404         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1405
1406         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Tmp(GPRInfo::regT0),
1407         Arg::widthArg(Width64));
1408
1409     int64_t things[2];
1410     Tmp base = code.newTmp(GP);
1411     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1412     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1413     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1414     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1415     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1416
1417     memset(things, 0, sizeof(things));
1418     
1419     CHECK(!compileAndRun<int>(proc));
1420
1421     CHECK(things[0] == 36000000000000ll);
1422     CHECK(things[1] == 1000000000000ll);
1423     CHECK(memory[0] == 2000000000000ll);
1424     CHECK(memory[1] == 35000000000000ll);
1425 }
1426
1427 void testShuffleRotateMemoryMixedWidth()
1428 {
1429     B3::Procedure proc;
1430     Code& code = proc.code();
1431
1432     int64_t memory[2];
1433     memory[0] = 35000000000000ll;
1434     memory[1] = 36000000000000ll;
1435
1436     BasicBlock* root = code.addBlock();
1437     loadConstant(root, 1000000000000ll, Tmp(GPRInfo::regT0));
1438     loadConstant(root, 2000000000000ll, Tmp(GPRInfo::regT1));
1439     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1440     root->append(
1441         Shuffle, nullptr,
1442         
1443         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1444
1445         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1446         Arg::widthArg(Width64),
1447         
1448         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1449         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Arg::widthArg(Width32),
1450
1451         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Tmp(GPRInfo::regT0),
1452         Arg::widthArg(Width64));
1453
1454     int64_t things[2];
1455     Tmp base = code.newTmp(GP);
1456     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1457     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1458     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1459     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1460     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1461
1462     memset(things, 0, sizeof(things));
1463     
1464     CHECK(!compileAndRun<int>(proc));
1465
1466     CHECK(things[0] == 36000000000000ll);
1467     CHECK(things[1] == static_cast<uint32_t>(1000000000000ll));
1468     CHECK(memory[0] == 2000000000000ll);
1469     CHECK(memory[1] == combineHiLo(36000000000000ll, 35000000000000ll));
1470 }
1471
1472 void testShuffleRotateMemoryAllRegs64()
1473 {
1474     B3::Procedure proc;
1475     Code& code = proc.code();
1476
1477     int64_t memory[2];
1478     memory[0] = 35000000000000ll;
1479     memory[1] = 36000000000000ll;
1480
1481     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1482     regs.removeFirst(Reg(GPRInfo::regT0));
1483
1484     BasicBlock* root = code.addBlock();
1485     for (unsigned i = 0; i < regs.size(); ++i)
1486         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1487     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1488     Inst& shuffle = root->append(
1489         Shuffle, nullptr,
1490         
1491         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1492         Arg::widthArg(Width64),
1493         
1494         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1495         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1496
1497         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1498         Arg::widthArg(Width64),
1499
1500         regs.last(), regs[0], Arg::widthArg(Width64));
1501
1502     for (unsigned i = 2; i < regs.size(); ++i)
1503         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1504
1505     Vector<int64_t> things(regs.size(), 666);
1506     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1507     for (unsigned i = 0; i < regs.size(); ++i) {
1508         root->append(
1509             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1510     }
1511     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1512     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1513
1514     CHECK(!compileAndRun<int>(proc));
1515
1516     CHECK(things[0] == static_cast<int64_t>(regs.size()) * 1000000000000ll);
1517     CHECK(things[1] == 36000000000000ll);
1518     for (unsigned i = 2; i < regs.size(); ++i)
1519         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1520     CHECK(memory[0] == 1000000000000ll);
1521     CHECK(memory[1] == 35000000000000ll);
1522 }
1523
1524 void testShuffleRotateMemoryAllRegsMixedWidth()
1525 {
1526     B3::Procedure proc;
1527     Code& code = proc.code();
1528
1529     int64_t memory[2];
1530     memory[0] = 35000000000000ll;
1531     memory[1] = 36000000000000ll;
1532
1533     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1534     regs.removeFirst(Reg(GPRInfo::regT0));
1535
1536     BasicBlock* root = code.addBlock();
1537     for (unsigned i = 0; i < regs.size(); ++i)
1538         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1539     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1540     Inst& shuffle = root->append(
1541         Shuffle, nullptr,
1542         
1543         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1544         Arg::widthArg(Width32),
1545         
1546         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1547         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1548
1549         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1550         Arg::widthArg(Width32),
1551
1552         regs.last(), regs[0], Arg::widthArg(Width32));
1553
1554     for (unsigned i = 2; i < regs.size(); ++i)
1555         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1556
1557     Vector<int64_t> things(regs.size(), 666);
1558     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1559     for (unsigned i = 0; i < regs.size(); ++i) {
1560         root->append(
1561             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1562     }
1563     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1564     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1565
1566     CHECK(!compileAndRun<int>(proc));
1567
1568     CHECK(things[0] == static_cast<uint32_t>(static_cast<int64_t>(regs.size()) * 1000000000000ll));
1569     CHECK(things[1] == static_cast<uint32_t>(36000000000000ll));
1570     for (unsigned i = 2; i < regs.size(); ++i)
1571         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1572     CHECK(memory[0] == combineHiLo(35000000000000ll, 1000000000000ll));
1573     CHECK(memory[1] == 35000000000000ll);
1574 }
1575
1576 void testShuffleSwapDouble()
1577 {
1578     B3::Procedure proc;
1579     Code& code = proc.code();
1580
1581     BasicBlock* root = code.addBlock();
1582     loadDoubleConstant(root, 1, Tmp(FPRInfo::fpRegT0), Tmp(GPRInfo::regT0));
1583     loadDoubleConstant(root, 2, Tmp(FPRInfo::fpRegT1), Tmp(GPRInfo::regT0));
1584     loadDoubleConstant(root, 3, Tmp(FPRInfo::fpRegT2), Tmp(GPRInfo::regT0));
1585     loadDoubleConstant(root, 4, Tmp(FPRInfo::fpRegT3), Tmp(GPRInfo::regT0));
1586     root->append(
1587         Shuffle, nullptr,
1588         Tmp(FPRInfo::fpRegT2), Tmp(FPRInfo::fpRegT3), Arg::widthArg(Width64),
1589         Tmp(FPRInfo::fpRegT3), Tmp(FPRInfo::fpRegT2), Arg::widthArg(Width64));
1590
1591     double things[4];
1592     Tmp base = code.newTmp(GP);
1593     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1594     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT0), Arg::addr(base, 0 * sizeof(double)));
1595     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT1), Arg::addr(base, 1 * sizeof(double)));
1596     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT2), Arg::addr(base, 2 * sizeof(double)));
1597     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT3), Arg::addr(base, 3 * sizeof(double)));
1598     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1599     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1600
1601     memset(things, 0, sizeof(things));
1602     
1603     CHECK(!compileAndRun<int>(proc));
1604
1605     CHECK(things[0] == 1);
1606     CHECK(things[1] == 2);
1607     CHECK(things[2] == 4);
1608     CHECK(things[3] == 3);
1609 }
1610
1611 void testShuffleShiftDouble()
1612 {
1613     B3::Procedure proc;
1614     Code& code = proc.code();
1615
1616     BasicBlock* root = code.addBlock();
1617     loadDoubleConstant(root, 1, Tmp(FPRInfo::fpRegT0), Tmp(GPRInfo::regT0));
1618     loadDoubleConstant(root, 2, Tmp(FPRInfo::fpRegT1), Tmp(GPRInfo::regT0));
1619     loadDoubleConstant(root, 3, Tmp(FPRInfo::fpRegT2), Tmp(GPRInfo::regT0));
1620     loadDoubleConstant(root, 4, Tmp(FPRInfo::fpRegT3), Tmp(GPRInfo::regT0));
1621     root->append(
1622         Shuffle, nullptr,
1623         Tmp(FPRInfo::fpRegT2), Tmp(FPRInfo::fpRegT3), Arg::widthArg(Width64));
1624
1625     double things[4];
1626     Tmp base = code.newTmp(GP);
1627     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1628     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT0), Arg::addr(base, 0 * sizeof(double)));
1629     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT1), Arg::addr(base, 1 * sizeof(double)));
1630     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT2), Arg::addr(base, 2 * sizeof(double)));
1631     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT3), Arg::addr(base, 3 * sizeof(double)));
1632     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1633     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1634
1635     memset(things, 0, sizeof(things));
1636     
1637     CHECK(!compileAndRun<int>(proc));
1638
1639     CHECK(things[0] == 1);
1640     CHECK(things[1] == 2);
1641     CHECK(things[2] == 3);
1642     CHECK(things[3] == 3);
1643 }
1644
1645 #if CPU(X86) || CPU(X86_64)
1646 void testX86VMULSD()
1647 {
1648     B3::Procedure proc;
1649     Code& code = proc.code();
1650
1651     BasicBlock* root = code.addBlock();
1652     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(FPRInfo::argumentFPR1), Tmp(FPRInfo::argumentFPR2));
1653     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1654     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1655
1656     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1657 }
1658
1659 void testX86VMULSDDestRex()
1660 {
1661     B3::Procedure proc;
1662     Code& code = proc.code();
1663
1664     BasicBlock* root = code.addBlock();
1665     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm15));
1666     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1667     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1668
1669     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1670 }
1671
1672 void testX86VMULSDOp1DestRex()
1673 {
1674     B3::Procedure proc;
1675     Code& code = proc.code();
1676
1677     BasicBlock* root = code.addBlock();
1678     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1679     root->append(MulDouble, nullptr, Tmp(X86Registers::xmm14), Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm15));
1680     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1681     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1682
1683     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1684 }
1685
1686 void testX86VMULSDOp2DestRex()
1687 {
1688     B3::Procedure proc;
1689     Code& code = proc.code();
1690
1691     BasicBlock* root = code.addBlock();
1692     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm14));
1693     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm15));
1694     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1695     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1696
1697     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1698 }
1699
1700 void testX86VMULSDOpsDestRex()
1701 {
1702     B3::Procedure proc;
1703     Code& code = proc.code();
1704
1705     BasicBlock* root = code.addBlock();
1706     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1707     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm13));
1708     root->append(MulDouble, nullptr, Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm13), Tmp(X86Registers::xmm15));
1709     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1710     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1711
1712     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1713 }
1714
1715 void testX86VMULSDAddr()
1716 {
1717     B3::Procedure proc;
1718     Code& code = proc.code();
1719
1720     BasicBlock* root = code.addBlock();
1721     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(GPRInfo::argumentGPR0), - 16), Tmp(FPRInfo::argumentFPR2));
1722     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1723     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1724
1725     double secondArg = 4.2;
1726     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 2, pureNaN()) == 2.4 * 4.2);
1727 }
1728
1729 void testX86VMULSDAddrOpRexAddr()
1730 {
1731     B3::Procedure proc;
1732     Code& code = proc.code();
1733
1734     BasicBlock* root = code.addBlock();
1735     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1736     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(X86Registers::r13), - 16), Tmp(FPRInfo::argumentFPR2));
1737     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1738     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1739
1740     double secondArg = 4.2;
1741     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 2, pureNaN()) == 2.4 * 4.2);
1742 }
1743
1744 void testX86VMULSDDestRexAddr()
1745 {
1746     B3::Procedure proc;
1747     Code& code = proc.code();
1748
1749     BasicBlock* root = code.addBlock();
1750     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(GPRInfo::argumentGPR0), 16), Tmp(X86Registers::xmm15));
1751     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1752     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1753
1754     double secondArg = 4.2;
1755     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 2, pureNaN()) == 2.4 * 4.2);
1756 }
1757
1758 void testX86VMULSDRegOpDestRexAddr()
1759 {
1760     B3::Procedure proc;
1761     Code& code = proc.code();
1762
1763     BasicBlock* root = code.addBlock();
1764     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1765     root->append(MulDouble, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0)), Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm15));
1766     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1767     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1768
1769     double secondArg = 4.2;
1770     CHECK(compileAndRun<double>(proc, 2.4, &secondArg, pureNaN()) == 2.4 * 4.2);
1771 }
1772
1773 void testX86VMULSDAddrOpDestRexAddr()
1774 {
1775     B3::Procedure proc;
1776     Code& code = proc.code();
1777
1778     BasicBlock* root = code.addBlock();
1779     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1780     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(X86Registers::r13), 8), Tmp(X86Registers::xmm15));
1781     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1782     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1783
1784     double secondArg = 4.2;
1785     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 1, pureNaN()) == 2.4 * 4.2);
1786 }
1787
1788 void testX86VMULSDBaseNeedsRex()
1789 {
1790     B3::Procedure proc;
1791     Code& code = proc.code();
1792
1793     BasicBlock* root = code.addBlock();
1794     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1795     root->append(MulDouble, nullptr, Arg::index(Tmp(X86Registers::r13), Tmp(GPRInfo::argumentGPR1)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1796     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1797     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1798
1799     double secondArg = 4.2;
1800     uint64_t index = 8;
1801     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 1, index, pureNaN()) == 2.4 * 4.2);
1802 }
1803
1804 void testX86VMULSDIndexNeedsRex()
1805 {
1806     B3::Procedure proc;
1807     Code& code = proc.code();
1808
1809     BasicBlock* root = code.addBlock();
1810     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR1), Tmp(X86Registers::r13));
1811     root->append(MulDouble, nullptr, Arg::index(Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1812     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1813     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1814
1815     double secondArg = 4.2;
1816     uint64_t index = - 8;
1817     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 1, index, pureNaN()) == 2.4 * 4.2);
1818 }
1819
1820 void testX86VMULSDBaseIndexNeedRex()
1821 {
1822     B3::Procedure proc;
1823     Code& code = proc.code();
1824
1825     BasicBlock* root = code.addBlock();
1826     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r12));
1827     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR1), Tmp(X86Registers::r13));
1828     root->append(MulDouble, nullptr, Arg::index(Tmp(X86Registers::r12), Tmp(X86Registers::r13)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1829     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1830     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1831
1832     double secondArg = 4.2;
1833     uint64_t index = 16;
1834     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 2, index, pureNaN()) == 2.4 * 4.2);
1835 }
1836 #endif // #if CPU(X86) || CPU(X86_64)
1837
1838 #if CPU(ARM64)
1839 void testInvalidateCachedTempRegisters()
1840 {
1841     B3::Procedure proc;
1842     Code& code = proc.code();
1843     BasicBlock* root = code.addBlock();
1844
1845     int32_t things[4];
1846     things[0] = 0x12000000;
1847     things[1] = 0x340000;
1848     things[2] = 0x5600;
1849     things[3] = 0x78;
1850     Tmp base = code.newTmp(GP);
1851     GPRReg tmp = GPRInfo::regT1;
1852     proc.pinRegister(tmp);
1853
1854     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1855
1856     B3::BasicBlock* patchPoint1Root = proc.addBlock();
1857     B3::Air::Special* patchpointSpecial = code.addSpecial(std::make_unique<B3::PatchpointSpecial>());
1858
1859     // In Patchpoint, Load things[0] -> tmp. This will materialize the address in x17 (dataMemoryRegister).
1860     B3::PatchpointValue* patchpoint1 = patchPoint1Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1861     patchpoint1->clobber(RegisterSet::macroScratchRegisters());
1862     patchpoint1->setGenerator(
1863         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1864             AllowMacroScratchRegisterUsage allowScratch(jit);
1865             jit.load32(&things, tmp);
1866         });
1867     root->append(Patch, patchpoint1, Arg::special(patchpointSpecial));
1868
1869     // Load things[1] -> x17, trashing dataMemoryRegister.
1870     root->append(Move32, nullptr, Arg::addr(base, 1 * sizeof(int32_t)), Tmp(ARM64Registers::x17));
1871     root->append(Add32, nullptr, Tmp(tmp), Tmp(ARM64Registers::x17), Tmp(GPRInfo::returnValueGPR));
1872
1873     // In Patchpoint, Load things[2] -> tmp. This should not reuse the prior contents of x17.
1874     B3::BasicBlock* patchPoint2Root = proc.addBlock();
1875     B3::PatchpointValue* patchpoint2 = patchPoint2Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1876     patchpoint2->clobber(RegisterSet::macroScratchRegisters());
1877     patchpoint2->setGenerator(
1878         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1879             AllowMacroScratchRegisterUsage allowScratch(jit);
1880             jit.load32(&things[2], tmp);
1881         });
1882     root->append(Patch, patchpoint2, Arg::special(patchpointSpecial));
1883
1884     root->append(Add32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR), Tmp(GPRInfo::returnValueGPR));
1885
1886     // In patchpoint, Store 0x78 -> things[3].
1887     // This will use and cache both x16 (dataMemoryRegister) and x17 (dataTempRegister).
1888     B3::BasicBlock* patchPoint3Root = proc.addBlock();
1889     B3::PatchpointValue* patchpoint3 = patchPoint3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1890     patchpoint3->clobber(RegisterSet::macroScratchRegisters());
1891     patchpoint3->setGenerator(
1892         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1893             AllowMacroScratchRegisterUsage allowScratch(jit);
1894             jit.store32(CCallHelpers::TrustedImm32(0x78), &things[3]);
1895         });
1896     root->append(Patch, patchpoint3, Arg::special(patchpointSpecial));
1897
1898     // Set x16 to 0xdead, trashing x16.
1899     root->append(Move, nullptr, Arg::bigImm(0xdead), Tmp(ARM64Registers::x16));
1900     root->append(Xor32, nullptr, Tmp(ARM64Registers::x16), Tmp(GPRInfo::returnValueGPR));
1901
1902     // In patchpoint, again Store 0x78 -> things[3].
1903     // This should rematerialize both x16 (dataMemoryRegister) and x17 (dataTempRegister).
1904     B3::BasicBlock* patchPoint4Root = proc.addBlock();
1905     B3::PatchpointValue* patchpoint4 = patchPoint4Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1906     patchpoint4->clobber(RegisterSet::macroScratchRegisters());
1907     patchpoint4->setGenerator(
1908         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1909             AllowMacroScratchRegisterUsage allowScratch(jit);
1910             jit.store32(CCallHelpers::TrustedImm32(0x78), &things[3]);
1911         });
1912     root->append(Patch, patchpoint4, Arg::special(patchpointSpecial));
1913
1914     root->append(Move, nullptr, Arg::bigImm(0xdead), Tmp(tmp));
1915     root->append(Xor32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR));
1916     root->append(Move32, nullptr, Arg::addr(base, 3 * sizeof(int32_t)), Tmp(tmp));
1917     root->append(Add32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR), Tmp(GPRInfo::returnValueGPR));
1918     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1919
1920     int32_t r = compileAndRun<int32_t>(proc);
1921     CHECK(r == 0x12345678);
1922 }
1923 #endif // #if CPU(ARM64)
1924
1925 void testArgumentRegPinned()
1926 {
1927     B3::Procedure proc;
1928     Code& code = proc.code();
1929     GPRReg pinned = GPRInfo::argumentGPR0;
1930     proc.pinRegister(pinned);
1931
1932     B3::Air::Special* patchpointSpecial = code.addSpecial(std::make_unique<B3::PatchpointSpecial>());
1933
1934     B3::BasicBlock* b3Root = proc.addBlock();
1935     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1936     patchpoint->clobber(RegisterSet(pinned));
1937     patchpoint->setGenerator(
1938         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1939             jit.move(CCallHelpers::TrustedImm32(42), pinned);
1940         });
1941
1942     BasicBlock* root = code.addBlock();
1943
1944     Tmp t1 = code.newTmp(GP);
1945     Tmp t2 = code.newTmp(GP);
1946
1947     root->append(Move, nullptr, Tmp(pinned), t1);
1948     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
1949     root->append(Move, nullptr, Tmp(pinned), t2);
1950     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
1951     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1952
1953     int32_t r = compileAndRun<int32_t>(proc, 10);
1954     CHECK(r == 10 + 42);
1955 }
1956
1957 void testArgumentRegPinned2()
1958 {
1959     B3::Procedure proc;
1960     Code& code = proc.code();
1961     GPRReg pinned = GPRInfo::argumentGPR0;
1962     proc.pinRegister(pinned);
1963
1964     B3::Air::Special* patchpointSpecial = code.addSpecial(std::make_unique<B3::PatchpointSpecial>());
1965
1966     B3::BasicBlock* b3Root = proc.addBlock();
1967     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1968     patchpoint->clobber({ }); 
1969     patchpoint->setGenerator(
1970         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1971             jit.move(CCallHelpers::TrustedImm32(42), pinned);
1972         });
1973
1974     BasicBlock* root = code.addBlock();
1975
1976     Tmp t1 = code.newTmp(GP);
1977     Tmp t2 = code.newTmp(GP);
1978
1979     // Since the patchpoint does not claim to clobber the pinned register,
1980     // the register allocator is allowed to either coalesce the first move,
1981     // the second move, or neither. The allowed results are:
1982     // - No move coalesced: 52
1983     // - The first move is coalesced: 84
1984     // - The second move is coalesced: 52
1985     root->append(Move, nullptr, Tmp(pinned), t1);
1986     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
1987     root->append(Move, nullptr, Tmp(pinned), t2);
1988     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
1989     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1990
1991     int32_t r = compileAndRun<int32_t>(proc, 10);
1992     CHECK(r == 52 || r == 84);
1993 }
1994
1995 void testArgumentRegPinned3()
1996 {
1997     B3::Procedure proc;
1998     Code& code = proc.code();
1999     GPRReg pinned = GPRInfo::argumentGPR0;
2000     proc.pinRegister(pinned);
2001
2002     B3::Air::Special* patchpointSpecial = code.addSpecial(std::make_unique<B3::PatchpointSpecial>());
2003
2004     B3::BasicBlock* b3Root = proc.addBlock();
2005     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
2006     patchpoint->clobber(RegisterSet(pinned));
2007     patchpoint->setGenerator(
2008         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2009             jit.move(CCallHelpers::TrustedImm32(42), pinned);
2010         });
2011
2012     BasicBlock* root = code.addBlock();
2013
2014     Tmp t1 = code.newTmp(GP);
2015     Tmp t2 = code.newTmp(GP);
2016     Tmp t3 = code.newTmp(GP);
2017
2018     root->append(Move, nullptr, Tmp(pinned), t1);
2019     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
2020     root->append(Move, nullptr, Tmp(pinned), t2);
2021     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
2022     root->append(Move, nullptr, Tmp(pinned), t3);
2023     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
2024     root->append(Add32, nullptr, Tmp(GPRInfo::returnValueGPR), t3, Tmp(GPRInfo::returnValueGPR));
2025     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
2026
2027     int32_t r = compileAndRun<int32_t>(proc, 10);
2028     CHECK(r == 10 + 42 + 42);
2029 }
2030
2031 void testLea64()
2032 {
2033     B3::Procedure proc;
2034     Code& code = proc.code();
2035
2036     BasicBlock* root = code.addBlock();
2037
2038     int64_t a = 0x11223344;
2039     int64_t b = 1 << 13;
2040
2041     root->append(Lea64, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0), b), Tmp(GPRInfo::returnValueGPR));
2042     root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));
2043
2044     int64_t r = compileAndRun<int64_t>(proc, a);
2045     CHECK(r == a + b);
2046 }
2047
2048 void testLea32()
2049 {
2050     B3::Procedure proc;
2051     Code& code = proc.code();
2052
2053     BasicBlock* root = code.addBlock();
2054
2055     int32_t a = 0x11223344;
2056     int32_t b = 1 << 13;
2057
2058     root->append(Lea32, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0), b), Tmp(GPRInfo::returnValueGPR));
2059     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
2060
2061     int32_t r = compileAndRun<int32_t>(proc, a);
2062     CHECK(r == a + b);
2063 }
2064
2065 #define PREFIX "O", Options::defaultB3OptLevel(), ": "
2066
2067 #define RUN(test) do {                                 \
2068         if (!shouldRun(#test))                         \
2069             break;                                     \
2070         tasks.append(                                  \
2071             createSharedTask<void()>(                  \
2072                 [&] () {                               \
2073                     dataLog(PREFIX #test "...\n");     \
2074                     test;                              \
2075                     dataLog(PREFIX #test ": OK!\n");   \
2076                 }));                                   \
2077     } while (false);
2078
2079 void run(const char* filter)
2080 {
2081     Deque<RefPtr<SharedTask<void()>>> tasks;
2082
2083     auto shouldRun = [&] (const char* testName) -> bool {
2084         return !filter || !!strcasestr(testName, filter);
2085     };
2086
2087     RUN(testSimple());
2088     
2089     RUN(testShuffleSimpleSwap());
2090     RUN(testShuffleSimpleShift());
2091     RUN(testShuffleLongShift());
2092     RUN(testShuffleLongShiftBackwards());
2093     RUN(testShuffleSimpleRotate());
2094     RUN(testShuffleSimpleBroadcast());
2095     RUN(testShuffleBroadcastAllRegs());
2096     RUN(testShuffleTreeShift());
2097     RUN(testShuffleTreeShiftBackward());
2098     RUN(testShuffleTreeShiftOtherBackward());
2099     RUN(testShuffleMultipleShifts());
2100     RUN(testShuffleRotateWithFringe());
2101     RUN(testShuffleRotateWithFringeInWeirdOrder());
2102     RUN(testShuffleRotateWithLongFringe());
2103     RUN(testShuffleMultipleRotates());
2104     RUN(testShuffleShiftAndRotate());
2105     RUN(testShuffleShiftAllRegs());
2106     RUN(testShuffleRotateAllRegs());
2107     RUN(testShuffleSimpleSwap64());
2108     RUN(testShuffleSimpleShift64());
2109     RUN(testShuffleSwapMixedWidth());
2110     RUN(testShuffleShiftMixedWidth());
2111     RUN(testShuffleShiftMemory());
2112     RUN(testShuffleShiftMemoryLong());
2113     RUN(testShuffleShiftMemoryAllRegs());
2114     RUN(testShuffleShiftMemoryAllRegs64());
2115     RUN(testShuffleShiftMemoryAllRegsMixedWidth());
2116     RUN(testShuffleRotateMemory());
2117     RUN(testShuffleRotateMemory64());
2118     RUN(testShuffleRotateMemoryMixedWidth());
2119     RUN(testShuffleRotateMemoryAllRegs64());
2120     RUN(testShuffleRotateMemoryAllRegsMixedWidth());
2121     RUN(testShuffleSwapDouble());
2122     RUN(testShuffleShiftDouble());
2123
2124 #if CPU(X86) || CPU(X86_64)
2125     RUN(testX86VMULSD());
2126     RUN(testX86VMULSDDestRex());
2127     RUN(testX86VMULSDOp1DestRex());
2128     RUN(testX86VMULSDOp2DestRex());
2129     RUN(testX86VMULSDOpsDestRex());
2130
2131     RUN(testX86VMULSDAddr());
2132     RUN(testX86VMULSDAddrOpRexAddr());
2133     RUN(testX86VMULSDDestRexAddr());
2134     RUN(testX86VMULSDRegOpDestRexAddr());
2135     RUN(testX86VMULSDAddrOpDestRexAddr());
2136
2137     RUN(testX86VMULSDBaseNeedsRex());
2138     RUN(testX86VMULSDIndexNeedsRex());
2139     RUN(testX86VMULSDBaseIndexNeedRex());
2140 #endif
2141
2142 #if CPU(ARM64)
2143     RUN(testInvalidateCachedTempRegisters());
2144 #endif
2145
2146     RUN(testArgumentRegPinned());
2147     RUN(testArgumentRegPinned2());
2148     RUN(testArgumentRegPinned3());
2149
2150     RUN(testLea32());
2151     RUN(testLea64());
2152
2153     if (tasks.isEmpty())
2154         usage();
2155
2156     Lock lock;
2157
2158     Vector<Ref<Thread>> threads;
2159     for (unsigned i = filter ? 1 : WTF::numberOfProcessorCores(); i--;) {
2160         threads.append(
2161             Thread::create(
2162                 "testair thread",
2163                 [&] () {
2164                     for (;;) {
2165                         RefPtr<SharedTask<void()>> task;
2166                         {
2167                             LockHolder locker(lock);
2168                             if (tasks.isEmpty())
2169                                 return;
2170                             task = tasks.takeFirst();
2171                         }
2172
2173                         task->run();
2174                     }
2175                 }));
2176     }
2177
2178     for (auto& thread : threads)
2179         thread->waitForCompletion();
2180     crashLock.lock();
2181     crashLock.unlock();
2182 }
2183
2184 } // anonymous namespace
2185
2186 #else // ENABLE(B3_JIT)
2187
2188 static void run(const char*)
2189 {
2190     dataLog("B3 JIT is not enabled.\n");
2191 }
2192
2193 #endif // ENABLE(B3_JIT)
2194
2195 int main(int argc, char** argv)
2196 {
2197     const char* filter = nullptr;
2198     switch (argc) {
2199     case 1:
2200         break;
2201     case 2:
2202         filter = argv[1];
2203         break;
2204     default:
2205         usage();
2206         break;
2207     }
2208     
2209     JSC::initializeThreading();
2210
2211     for (unsigned i = 0; i <= 2; ++i) {
2212         JSC::Options::defaultB3OptLevel() = i;
2213         run(filter);
2214     }
2215
2216     return 0;
2217 }