[WTF] Add makeUnique<T>, which ensures T is fast-allocated, makeUnique / makeUniqueWi...
[WebKit-https.git] / Source / JavaScriptCore / b3 / air / testair.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #include "AirCode.h"
29 #include "AirGenerate.h"
30 #include "AirInstInlines.h"
31 #include "AirSpecial.h"
32 #include "AllowMacroScratchRegisterUsage.h"
33 #include "B3BasicBlockInlines.h"
34 #include "B3Compilation.h"
35 #include "B3Procedure.h"
36 #include "B3PatchpointSpecial.h"
37 #include "CCallHelpers.h"
38 #include "InitializeThreading.h"
39 #include "JSCInlines.h"
40 #include "LinkBuffer.h"
41 #include "PureNaN.h"
42 #include <cmath>
43 #include <string>
44 #include <wtf/Lock.h>
45 #include <wtf/NumberOfCores.h>
46 #include <wtf/StdMap.h>
47 #include <wtf/Threading.h>
48 #include <wtf/text/StringCommon.h>
49
50 // We don't have a NO_RETURN_DUE_TO_EXIT, nor should we. That's ridiculous.
51 static bool hiddenTruthBecauseNoReturnIsStupid() { return true; }
52
53 static void usage()
54 {
55     dataLog("Usage: testair [<filter>]\n");
56     if (hiddenTruthBecauseNoReturnIsStupid())
57         exit(1);
58 }
59
60 #if ENABLE(B3_JIT)
61
62 using namespace JSC;
63 using namespace JSC::B3::Air;
64
65 using JSC::B3::FP;
66 using JSC::B3::GP;
67 using JSC::B3::Width;
68 using JSC::B3::Width8;
69 using JSC::B3::Width16;
70 using JSC::B3::Width32;
71 using JSC::B3::Width64;
72
73 namespace {
74
75 Lock crashLock;
76
77 // Nothing fancy for now; we just use the existing WTF assertion machinery.
78 #define CHECK(x) do {                                                   \
79         if (!!(x))                                                      \
80             break;                                                      \
81         crashLock.lock();                                               \
82         WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #x); \
83         CRASH();                                                        \
84     } while (false)
85
86 std::unique_ptr<B3::Compilation> compile(B3::Procedure& proc)
87 {
88     prepareForGeneration(proc.code());
89     CCallHelpers jit;
90     generate(proc.code(), jit);
91     LinkBuffer linkBuffer(jit, nullptr);
92
93     return makeUnique<B3::Compilation>(
94         FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testair compilation"), proc.releaseByproducts());
95 }
96
97 template<typename T, typename... Arguments>
98 T invoke(const B3::Compilation& code, Arguments... arguments)
99 {
100     void* executableAddress = untagCFunctionPtr(code.code().executableAddress(), B3CompilationPtrTag);
101     T (*function)(Arguments...) = bitwise_cast<T(*)(Arguments...)>(executableAddress);
102     return function(arguments...);
103 }
104
105 template<typename T, typename... Arguments>
106 T compileAndRun(B3::Procedure& procedure, Arguments... arguments)
107 {
108     return invoke<T>(*compile(procedure), arguments...);
109 }
110
111 void testSimple()
112 {
113     B3::Procedure proc;
114     Code& code = proc.code();
115
116     BasicBlock* root = code.addBlock();
117     root->append(Move, nullptr, Arg::imm(42), Tmp(GPRInfo::returnValueGPR));
118     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
119
120     CHECK(compileAndRun<int>(proc) == 42);
121 }
122
123 // Use this to put a constant into a register without Air being able to see the constant.
124 template<typename T>
125 void loadConstantImpl(BasicBlock* block, T value, B3::Air::Opcode move, Tmp tmp, Tmp scratch)
126 {
127     static Lock lock;
128     static StdMap<T, T*>* map; // I'm not messing with HashMap's problems with integers.
129
130     LockHolder locker(lock);
131     if (!map)
132         map = new StdMap<T, T*>();
133
134     if (!map->count(value))
135         (*map)[value] = new T(value);
136
137     T* ptr = (*map)[value];
138     block->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(ptr)), scratch);
139     block->append(move, nullptr, Arg::addr(scratch), tmp);
140 }
141
142 template<typename T>
143 void loadConstant(BasicBlock* block, T value, Tmp tmp)
144 {
145     loadConstantImpl(block, value, Move, tmp, tmp);
146 }
147
148 void loadDoubleConstant(BasicBlock* block, double value, Tmp tmp, Tmp scratch)
149 {
150     loadConstantImpl<double>(block, value, MoveDouble, tmp, scratch);
151 }
152
153 void testShuffleSimpleSwap()
154 {
155     B3::Procedure proc;
156     Code& code = proc.code();
157
158     BasicBlock* root = code.addBlock();
159     loadConstant(root, 1, Tmp(GPRInfo::regT0));
160     loadConstant(root, 2, Tmp(GPRInfo::regT1));
161     loadConstant(root, 3, Tmp(GPRInfo::regT2));
162     loadConstant(root, 4, Tmp(GPRInfo::regT3));
163     root->append(
164         Shuffle, nullptr,
165         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
166         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width32));
167
168     int32_t things[4];
169     Tmp base = code.newTmp(GP);
170     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
171     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
172     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
173     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
174     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
175     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
176     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
177
178     memset(things, 0, sizeof(things));
179     
180     CHECK(!compileAndRun<int>(proc));
181
182     CHECK(things[0] == 1);
183     CHECK(things[1] == 2);
184     CHECK(things[2] == 4);
185     CHECK(things[3] == 3);
186 }
187
188 void testShuffleSimpleShift()
189 {
190     B3::Procedure proc;
191     Code& code = proc.code();
192
193     BasicBlock* root = code.addBlock();
194     loadConstant(root, 1, Tmp(GPRInfo::regT0));
195     loadConstant(root, 2, Tmp(GPRInfo::regT1));
196     loadConstant(root, 3, Tmp(GPRInfo::regT2));
197     loadConstant(root, 4, Tmp(GPRInfo::regT3));
198     root->append(
199         Shuffle, nullptr,
200         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
201         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32));
202
203     int32_t things[5];
204     Tmp base = code.newTmp(GP);
205     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
206     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
207     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
208     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
209     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
210     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
211     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
212     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
213
214     memset(things, 0, sizeof(things));
215     
216     CHECK(!compileAndRun<int>(proc));
217
218     CHECK(things[0] == 1);
219     CHECK(things[1] == 2);
220     CHECK(things[2] == 3);
221     CHECK(things[3] == 3);
222     CHECK(things[4] == 4);
223 }
224
225 void testShuffleLongShift()
226 {
227     B3::Procedure proc;
228     Code& code = proc.code();
229
230     BasicBlock* root = code.addBlock();
231     loadConstant(root, 1, Tmp(GPRInfo::regT0));
232     loadConstant(root, 2, Tmp(GPRInfo::regT1));
233     loadConstant(root, 3, Tmp(GPRInfo::regT2));
234     loadConstant(root, 4, Tmp(GPRInfo::regT3));
235     loadConstant(root, 5, Tmp(GPRInfo::regT4));
236     loadConstant(root, 6, Tmp(GPRInfo::regT5));
237     loadConstant(root, 7, Tmp(GPRInfo::regT6));
238     loadConstant(root, 8, Tmp(GPRInfo::regT7));
239     root->append(
240         Shuffle, nullptr,
241         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
242         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
243         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
244         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
245         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
246         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
247         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT7), Arg::widthArg(Width32));
248
249     int32_t things[8];
250     Tmp base = code.newTmp(GP);
251     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
252     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
253     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
254     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
255     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
256     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
257     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
258     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
259     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
260     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
261     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
262
263     memset(things, 0, sizeof(things));
264     
265     CHECK(!compileAndRun<int>(proc));
266
267     CHECK(things[0] == 1);
268     CHECK(things[1] == 1);
269     CHECK(things[2] == 2);
270     CHECK(things[3] == 3);
271     CHECK(things[4] == 4);
272     CHECK(things[5] == 5);
273     CHECK(things[6] == 6);
274     CHECK(things[7] == 7);
275 }
276
277 void testShuffleLongShiftBackwards()
278 {
279     B3::Procedure proc;
280     Code& code = proc.code();
281
282     BasicBlock* root = code.addBlock();
283     loadConstant(root, 1, Tmp(GPRInfo::regT0));
284     loadConstant(root, 2, Tmp(GPRInfo::regT1));
285     loadConstant(root, 3, Tmp(GPRInfo::regT2));
286     loadConstant(root, 4, Tmp(GPRInfo::regT3));
287     loadConstant(root, 5, Tmp(GPRInfo::regT4));
288     loadConstant(root, 6, Tmp(GPRInfo::regT5));
289     loadConstant(root, 7, Tmp(GPRInfo::regT6));
290     loadConstant(root, 8, Tmp(GPRInfo::regT7));
291     root->append(
292         Shuffle, nullptr,
293         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
294         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
295         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
296         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
297         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
298         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
299         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
300
301     int32_t things[8];
302     Tmp base = code.newTmp(GP);
303     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
304     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
305     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
306     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
307     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
308     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
309     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
310     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
311     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
312     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
313     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
314
315     memset(things, 0, sizeof(things));
316     
317     CHECK(!compileAndRun<int>(proc));
318
319     CHECK(things[0] == 1);
320     CHECK(things[1] == 1);
321     CHECK(things[2] == 2);
322     CHECK(things[3] == 3);
323     CHECK(things[4] == 4);
324     CHECK(things[5] == 5);
325     CHECK(things[6] == 6);
326     CHECK(things[7] == 7);
327 }
328
329 void testShuffleSimpleRotate()
330 {
331     B3::Procedure proc;
332     Code& code = proc.code();
333
334     BasicBlock* root = code.addBlock();
335     loadConstant(root, 1, Tmp(GPRInfo::regT0));
336     loadConstant(root, 2, Tmp(GPRInfo::regT1));
337     loadConstant(root, 3, Tmp(GPRInfo::regT2));
338     loadConstant(root, 4, Tmp(GPRInfo::regT3));
339     root->append(
340         Shuffle, nullptr,
341         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
342         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
343         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32));
344
345     int32_t things[4];
346     Tmp base = code.newTmp(GP);
347     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
348     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
349     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
350     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
351     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
352     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
353     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
354
355     memset(things, 0, sizeof(things));
356     
357     CHECK(!compileAndRun<int>(proc));
358
359     CHECK(things[0] == 3);
360     CHECK(things[1] == 1);
361     CHECK(things[2] == 2);
362     CHECK(things[3] == 4);
363 }
364
365 void testShuffleSimpleBroadcast()
366 {
367     B3::Procedure proc;
368     Code& code = proc.code();
369
370     BasicBlock* root = code.addBlock();
371     loadConstant(root, 1, Tmp(GPRInfo::regT0));
372     loadConstant(root, 2, Tmp(GPRInfo::regT1));
373     loadConstant(root, 3, Tmp(GPRInfo::regT2));
374     loadConstant(root, 4, Tmp(GPRInfo::regT3));
375     root->append(
376         Shuffle, nullptr,
377         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
378         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
379         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32));
380
381     int32_t things[4];
382     Tmp base = code.newTmp(GP);
383     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
384     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
385     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
386     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
387     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
388     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
389     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
390
391     memset(things, 0, sizeof(things));
392     
393     CHECK(!compileAndRun<int>(proc));
394
395     CHECK(things[0] == 1);
396     CHECK(things[1] == 1);
397     CHECK(things[2] == 1);
398     CHECK(things[3] == 1);
399 }
400
401 void testShuffleBroadcastAllRegs()
402 {
403     B3::Procedure proc;
404     Code& code = proc.code();
405
406     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
407
408     BasicBlock* root = code.addBlock();
409     root->append(Move, nullptr, Arg::imm(35), Tmp(GPRInfo::regT0));
410     unsigned count = 1;
411     for (Reg reg : regs) {
412         if (reg != Reg(GPRInfo::regT0))
413             loadConstant(root, count++, Tmp(reg));
414     }
415     Inst& shuffle = root->append(Shuffle, nullptr);
416     for (Reg reg : regs) {
417         if (reg != Reg(GPRInfo::regT0))
418             shuffle.append(Tmp(GPRInfo::regT0), Tmp(reg), Arg::widthArg(Width32));
419     }
420
421     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
422     for (unsigned i = 0; i < regs.size(); ++i)
423         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
424
425     Vector<int32_t> things(regs.size(), 666);
426     Tmp base = code.newTmp(GP);
427     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
428     for (unsigned i = 0; i < regs.size(); ++i) {
429         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
430         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
431     }
432     
433     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
434     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
435
436     CHECK(!compileAndRun<int>(proc));
437
438     for (int32_t thing : things)
439         CHECK(thing == 35);
440 }
441
442 void testShuffleTreeShift()
443 {
444     B3::Procedure proc;
445     Code& code = proc.code();
446
447     BasicBlock* root = code.addBlock();
448     loadConstant(root, 1, Tmp(GPRInfo::regT0));
449     loadConstant(root, 2, Tmp(GPRInfo::regT1));
450     loadConstant(root, 3, Tmp(GPRInfo::regT2));
451     loadConstant(root, 4, Tmp(GPRInfo::regT3));
452     loadConstant(root, 5, Tmp(GPRInfo::regT4));
453     loadConstant(root, 6, Tmp(GPRInfo::regT5));
454     loadConstant(root, 7, Tmp(GPRInfo::regT6));
455     loadConstant(root, 8, Tmp(GPRInfo::regT7));
456     root->append(
457         Shuffle, nullptr,
458         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
459         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
460         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
461         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
462         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
463         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
464         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT7), Arg::widthArg(Width32));
465
466     int32_t things[8];
467     Tmp base = code.newTmp(GP);
468     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
469     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
470     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
471     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
472     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
473     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
474     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
475     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
476     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
477     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
478     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
479
480     memset(things, 0, sizeof(things));
481     
482     CHECK(!compileAndRun<int>(proc));
483
484     CHECK(things[0] == 1);
485     CHECK(things[1] == 1);
486     CHECK(things[2] == 1);
487     CHECK(things[3] == 2);
488     CHECK(things[4] == 2);
489     CHECK(things[5] == 3);
490     CHECK(things[6] == 3);
491     CHECK(things[7] == 4);
492 }
493
494 void testShuffleTreeShiftBackward()
495 {
496     B3::Procedure proc;
497     Code& code = proc.code();
498
499     BasicBlock* root = code.addBlock();
500     loadConstant(root, 1, Tmp(GPRInfo::regT0));
501     loadConstant(root, 2, Tmp(GPRInfo::regT1));
502     loadConstant(root, 3, Tmp(GPRInfo::regT2));
503     loadConstant(root, 4, Tmp(GPRInfo::regT3));
504     loadConstant(root, 5, Tmp(GPRInfo::regT4));
505     loadConstant(root, 6, Tmp(GPRInfo::regT5));
506     loadConstant(root, 7, Tmp(GPRInfo::regT6));
507     loadConstant(root, 8, Tmp(GPRInfo::regT7));
508     root->append(
509         Shuffle, nullptr,
510         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
511         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
512         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
513         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
514         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
515         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
516         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
517
518     int32_t things[8];
519     Tmp base = code.newTmp(GP);
520     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
521     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
522     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
523     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
524     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
525     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
526     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
527     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
528     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
529     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
530     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
531
532     memset(things, 0, sizeof(things));
533     
534     CHECK(!compileAndRun<int>(proc));
535
536     CHECK(things[0] == 1);
537     CHECK(things[1] == 1);
538     CHECK(things[2] == 1);
539     CHECK(things[3] == 2);
540     CHECK(things[4] == 2);
541     CHECK(things[5] == 3);
542     CHECK(things[6] == 3);
543     CHECK(things[7] == 4);
544 }
545
546 void testShuffleTreeShiftOtherBackward()
547 {
548     // NOTE: This test was my original attempt at TreeShiftBackward but mistakes were made. So, this
549     // ends up being just a weird test. But weird tests are useful, so I kept it.
550     
551     B3::Procedure proc;
552     Code& code = proc.code();
553
554     BasicBlock* root = code.addBlock();
555     loadConstant(root, 1, Tmp(GPRInfo::regT0));
556     loadConstant(root, 2, Tmp(GPRInfo::regT1));
557     loadConstant(root, 3, Tmp(GPRInfo::regT2));
558     loadConstant(root, 4, Tmp(GPRInfo::regT3));
559     loadConstant(root, 5, Tmp(GPRInfo::regT4));
560     loadConstant(root, 6, Tmp(GPRInfo::regT5));
561     loadConstant(root, 7, Tmp(GPRInfo::regT6));
562     loadConstant(root, 8, Tmp(GPRInfo::regT7));
563     root->append(
564         Shuffle, nullptr,
565         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT7), Arg::widthArg(Width32),
566         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT6), Arg::widthArg(Width32),
567         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
568         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
569         Tmp(GPRInfo::regT6), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
570         Tmp(GPRInfo::regT7), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
571         Tmp(GPRInfo::regT7), Tmp(GPRInfo::regT1), Arg::widthArg(Width32));
572
573     int32_t things[8];
574     Tmp base = code.newTmp(GP);
575     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
576     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
577     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
578     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
579     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
580     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
581     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
582     root->append(Move32, nullptr, Tmp(GPRInfo::regT6), Arg::addr(base, 6 * sizeof(int32_t)));
583     root->append(Move32, nullptr, Tmp(GPRInfo::regT7), Arg::addr(base, 7 * sizeof(int32_t)));
584     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
585     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
586
587     memset(things, 0, sizeof(things));
588     
589     CHECK(!compileAndRun<int>(proc));
590
591     CHECK(things[0] == 1);
592     CHECK(things[1] == 8);
593     CHECK(things[2] == 8);
594     CHECK(things[3] == 7);
595     CHECK(things[4] == 7);
596     CHECK(things[5] == 6);
597     CHECK(things[6] == 6);
598     CHECK(things[7] == 5);
599 }
600
601 void testShuffleMultipleShifts()
602 {
603     B3::Procedure proc;
604     Code& code = proc.code();
605
606     BasicBlock* root = code.addBlock();
607     loadConstant(root, 1, Tmp(GPRInfo::regT0));
608     loadConstant(root, 2, Tmp(GPRInfo::regT1));
609     loadConstant(root, 3, Tmp(GPRInfo::regT2));
610     loadConstant(root, 4, Tmp(GPRInfo::regT3));
611     loadConstant(root, 5, Tmp(GPRInfo::regT4));
612     loadConstant(root, 6, Tmp(GPRInfo::regT5));
613     root->append(
614         Shuffle, nullptr,
615         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
616         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
617         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
618         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
619
620     int32_t things[6];
621     Tmp base = code.newTmp(GP);
622     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
623     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
624     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
625     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
626     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
627     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
628     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
629     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
630     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
631
632     memset(things, 0, sizeof(things));
633     
634     CHECK(!compileAndRun<int>(proc));
635
636     CHECK(things[0] == 1);
637     CHECK(things[1] == 1);
638     CHECK(things[2] == 3);
639     CHECK(things[3] == 3);
640     CHECK(things[4] == 3);
641     CHECK(things[5] == 1);
642 }
643
644 void testShuffleRotateWithFringe()
645 {
646     B3::Procedure proc;
647     Code& code = proc.code();
648
649     BasicBlock* root = code.addBlock();
650     loadConstant(root, 1, Tmp(GPRInfo::regT0));
651     loadConstant(root, 2, Tmp(GPRInfo::regT1));
652     loadConstant(root, 3, Tmp(GPRInfo::regT2));
653     loadConstant(root, 4, Tmp(GPRInfo::regT3));
654     loadConstant(root, 5, Tmp(GPRInfo::regT4));
655     loadConstant(root, 6, Tmp(GPRInfo::regT5));
656     root->append(
657         Shuffle, nullptr,
658         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
659         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
660         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
661         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
662         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
663         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
664
665     int32_t things[6];
666     Tmp base = code.newTmp(GP);
667     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
668     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
669     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
670     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
671     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
672     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
673     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
674     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
675     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
676
677     memset(things, 0, sizeof(things));
678     
679     CHECK(!compileAndRun<int>(proc));
680
681     CHECK(things[0] == 3);
682     CHECK(things[1] == 1);
683     CHECK(things[2] == 2);
684     CHECK(things[3] == 1);
685     CHECK(things[4] == 2);
686     CHECK(things[5] == 3);
687 }
688
689 void testShuffleRotateWithFringeInWeirdOrder()
690 {
691     B3::Procedure proc;
692     Code& code = proc.code();
693
694     BasicBlock* root = code.addBlock();
695     loadConstant(root, 1, Tmp(GPRInfo::regT0));
696     loadConstant(root, 2, Tmp(GPRInfo::regT1));
697     loadConstant(root, 3, Tmp(GPRInfo::regT2));
698     loadConstant(root, 4, Tmp(GPRInfo::regT3));
699     loadConstant(root, 5, Tmp(GPRInfo::regT4));
700     loadConstant(root, 6, Tmp(GPRInfo::regT5));
701     root->append(
702         Shuffle, nullptr,
703         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
704         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
705         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
706         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
707         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
708         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32));
709
710     int32_t things[6];
711     Tmp base = code.newTmp(GP);
712     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
713     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
714     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
715     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
716     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
717     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
718     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
719     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
720     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
721
722     memset(things, 0, sizeof(things));
723     
724     CHECK(!compileAndRun<int>(proc));
725
726     CHECK(things[0] == 3);
727     CHECK(things[1] == 1);
728     CHECK(things[2] == 2);
729     CHECK(things[3] == 1);
730     CHECK(things[4] == 2);
731     CHECK(things[5] == 3);
732 }
733
734 void testShuffleRotateWithLongFringe()
735 {
736     B3::Procedure proc;
737     Code& code = proc.code();
738
739     BasicBlock* root = code.addBlock();
740     loadConstant(root, 1, Tmp(GPRInfo::regT0));
741     loadConstant(root, 2, Tmp(GPRInfo::regT1));
742     loadConstant(root, 3, Tmp(GPRInfo::regT2));
743     loadConstant(root, 4, Tmp(GPRInfo::regT3));
744     loadConstant(root, 5, Tmp(GPRInfo::regT4));
745     loadConstant(root, 6, Tmp(GPRInfo::regT5));
746     root->append(
747         Shuffle, nullptr,
748         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
749         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
750         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
751         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
752         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
753         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
754
755     int32_t things[6];
756     Tmp base = code.newTmp(GP);
757     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
758     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
759     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
760     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
761     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
762     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
763     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
764     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
765     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
766
767     memset(things, 0, sizeof(things));
768     
769     CHECK(!compileAndRun<int>(proc));
770
771     CHECK(things[0] == 3);
772     CHECK(things[1] == 1);
773     CHECK(things[2] == 2);
774     CHECK(things[3] == 1);
775     CHECK(things[4] == 4);
776     CHECK(things[5] == 5);
777 }
778
779 void testShuffleMultipleRotates()
780 {
781     B3::Procedure proc;
782     Code& code = proc.code();
783
784     BasicBlock* root = code.addBlock();
785     loadConstant(root, 1, Tmp(GPRInfo::regT0));
786     loadConstant(root, 2, Tmp(GPRInfo::regT1));
787     loadConstant(root, 3, Tmp(GPRInfo::regT2));
788     loadConstant(root, 4, Tmp(GPRInfo::regT3));
789     loadConstant(root, 5, Tmp(GPRInfo::regT4));
790     loadConstant(root, 6, Tmp(GPRInfo::regT5));
791     root->append(
792         Shuffle, nullptr,
793         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
794         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
795         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
796         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
797         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32),
798         Tmp(GPRInfo::regT5), Tmp(GPRInfo::regT3), Arg::widthArg(Width32));
799
800     int32_t things[6];
801     Tmp base = code.newTmp(GP);
802     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
803     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
804     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
805     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
806     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
807     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
808     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
809     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
810     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
811
812     memset(things, 0, sizeof(things));
813     
814     CHECK(!compileAndRun<int>(proc));
815
816     CHECK(things[0] == 3);
817     CHECK(things[1] == 1);
818     CHECK(things[2] == 2);
819     CHECK(things[3] == 6);
820     CHECK(things[4] == 4);
821     CHECK(things[5] == 5);
822 }
823
824 void testShuffleShiftAndRotate()
825 {
826     B3::Procedure proc;
827     Code& code = proc.code();
828
829     BasicBlock* root = code.addBlock();
830     loadConstant(root, 1, Tmp(GPRInfo::regT0));
831     loadConstant(root, 2, Tmp(GPRInfo::regT1));
832     loadConstant(root, 3, Tmp(GPRInfo::regT2));
833     loadConstant(root, 4, Tmp(GPRInfo::regT3));
834     loadConstant(root, 5, Tmp(GPRInfo::regT4));
835     loadConstant(root, 6, Tmp(GPRInfo::regT5));
836     root->append(
837         Shuffle, nullptr,
838         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
839         Tmp(GPRInfo::regT1), Tmp(GPRInfo::regT2), Arg::widthArg(Width32),
840         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT0), Arg::widthArg(Width32),
841         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32),
842         Tmp(GPRInfo::regT4), Tmp(GPRInfo::regT5), Arg::widthArg(Width32));
843
844     int32_t things[6];
845     Tmp base = code.newTmp(GP);
846     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
847     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
848     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
849     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
850     root->append(Move32, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int32_t)));
851     root->append(Move32, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int32_t)));
852     root->append(Move32, nullptr, Tmp(GPRInfo::regT5), Arg::addr(base, 5 * sizeof(int32_t)));
853     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
854     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
855
856     memset(things, 0, sizeof(things));
857     
858     CHECK(!compileAndRun<int>(proc));
859
860     CHECK(things[0] == 3);
861     CHECK(things[1] == 1);
862     CHECK(things[2] == 2);
863     CHECK(things[3] == 4);
864     CHECK(things[4] == 4);
865     CHECK(things[5] == 5);
866 }
867
868 void testShuffleShiftAllRegs()
869 {
870     B3::Procedure proc;
871     Code& code = proc.code();
872
873     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
874
875     BasicBlock* root = code.addBlock();
876     for (unsigned i = 0; i < regs.size(); ++i)
877         loadConstant(root, 35 + i, Tmp(regs[i]));
878     Inst& shuffle = root->append(Shuffle, nullptr);
879     for (unsigned i = 1; i < regs.size(); ++i)
880         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
881
882     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
883     for (unsigned i = 0; i < regs.size(); ++i)
884         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
885
886     Vector<int32_t> things(regs.size(), 666);
887     Tmp base = code.newTmp(GP);
888     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
889     for (unsigned i = 0; i < regs.size(); ++i) {
890         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
891         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
892     }
893     
894     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
895     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
896
897     CHECK(!compileAndRun<int>(proc));
898
899     CHECK(things[0] == 35);
900     for (unsigned i = 1; i < regs.size(); ++i)
901         CHECK(things[i] == 35 + static_cast<int32_t>(i) - 1);
902 }
903
904 void testShuffleRotateAllRegs()
905 {
906     B3::Procedure proc;
907     Code& code = proc.code();
908
909     const Vector<Reg>& regs = code.regsInPriorityOrder(GP);
910
911     BasicBlock* root = code.addBlock();
912     for (unsigned i = 0; i < regs.size(); ++i)
913         loadConstant(root, 35 + i, Tmp(regs[i]));
914     Inst& shuffle = root->append(Shuffle, nullptr);
915     for (unsigned i = 1; i < regs.size(); ++i)
916         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
917     shuffle.append(Tmp(regs.last()), Tmp(regs[0]), Arg::widthArg(Width32));
918
919     StackSlot* slot = code.addStackSlot(sizeof(int32_t) * regs.size(), StackSlotKind::Locked);
920     for (unsigned i = 0; i < regs.size(); ++i)
921         root->append(Move32, nullptr, Tmp(regs[i]), Arg::stack(slot, i * sizeof(int32_t)));
922
923     Vector<int32_t> things(regs.size(), 666);
924     Tmp base = code.newTmp(GP);
925     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), base);
926     for (unsigned i = 0; i < regs.size(); ++i) {
927         root->append(Move32, nullptr, Arg::stack(slot, i * sizeof(int32_t)), Tmp(GPRInfo::regT0));
928         root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, i * sizeof(int32_t)));
929     }
930     
931     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
932     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
933
934     CHECK(!compileAndRun<int>(proc));
935
936     CHECK(things[0] == 35 + static_cast<int32_t>(regs.size()) - 1);
937     for (unsigned i = 1; i < regs.size(); ++i)
938         CHECK(things[i] == 35 + static_cast<int32_t>(i) - 1);
939 }
940
941 void testShuffleSimpleSwap64()
942 {
943     B3::Procedure proc;
944     Code& code = proc.code();
945
946     BasicBlock* root = code.addBlock();
947     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
948     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
949     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
950     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
951     root->append(
952         Shuffle, nullptr,
953         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
954         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width64));
955
956     int64_t things[4];
957     Tmp base = code.newTmp(GP);
958     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
959     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
960     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
961     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
962     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
963     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
964     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
965
966     memset(things, 0, sizeof(things));
967     
968     CHECK(!compileAndRun<int>(proc));
969
970     CHECK(things[0] == 10000000000000000ll);
971     CHECK(things[1] == 20000000000000000ll);
972     CHECK(things[2] == 40000000000000000ll);
973     CHECK(things[3] == 30000000000000000ll);
974 }
975
976 void testShuffleSimpleShift64()
977 {
978     B3::Procedure proc;
979     Code& code = proc.code();
980
981     BasicBlock* root = code.addBlock();
982     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
983     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
984     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
985     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
986     loadConstant(root, 50000000000000000ll, Tmp(GPRInfo::regT4));
987     root->append(
988         Shuffle, nullptr,
989         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
990         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width64));
991
992     int64_t things[5];
993     Tmp base = code.newTmp(GP);
994     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
995     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
996     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
997     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
998     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
999     root->append(Move, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int64_t)));
1000     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1001     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1002
1003     memset(things, 0, sizeof(things));
1004     
1005     CHECK(!compileAndRun<int>(proc));
1006
1007     CHECK(things[0] == 10000000000000000ll);
1008     CHECK(things[1] == 20000000000000000ll);
1009     CHECK(things[2] == 30000000000000000ll);
1010     CHECK(things[3] == 30000000000000000ll);
1011     CHECK(things[4] == 40000000000000000ll);
1012 }
1013
1014 void testShuffleSwapMixedWidth()
1015 {
1016     B3::Procedure proc;
1017     Code& code = proc.code();
1018
1019     BasicBlock* root = code.addBlock();
1020     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
1021     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
1022     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
1023     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
1024     root->append(
1025         Shuffle, nullptr,
1026         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width32),
1027         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT2), Arg::widthArg(Width64));
1028
1029     int64_t things[4];
1030     Tmp base = code.newTmp(GP);
1031     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1032     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1033     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1034     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
1035     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
1036     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1037     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1038
1039     memset(things, 0, sizeof(things));
1040     
1041     CHECK(!compileAndRun<int>(proc));
1042
1043     CHECK(things[0] == 10000000000000000ll);
1044     CHECK(things[1] == 20000000000000000ll);
1045     CHECK(things[2] == 40000000000000000ll);
1046     CHECK(things[3] == static_cast<uint32_t>(30000000000000000ll));
1047 }
1048
1049 void testShuffleShiftMixedWidth()
1050 {
1051     B3::Procedure proc;
1052     Code& code = proc.code();
1053
1054     BasicBlock* root = code.addBlock();
1055     loadConstant(root, 10000000000000000ll, Tmp(GPRInfo::regT0));
1056     loadConstant(root, 20000000000000000ll, Tmp(GPRInfo::regT1));
1057     loadConstant(root, 30000000000000000ll, Tmp(GPRInfo::regT2));
1058     loadConstant(root, 40000000000000000ll, Tmp(GPRInfo::regT3));
1059     loadConstant(root, 50000000000000000ll, Tmp(GPRInfo::regT4));
1060     root->append(
1061         Shuffle, nullptr,
1062         Tmp(GPRInfo::regT2), Tmp(GPRInfo::regT3), Arg::widthArg(Width64),
1063         Tmp(GPRInfo::regT3), Tmp(GPRInfo::regT4), Arg::widthArg(Width32));
1064
1065     int64_t things[5];
1066     Tmp base = code.newTmp(GP);
1067     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1068     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1069     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1070     root->append(Move, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int64_t)));
1071     root->append(Move, nullptr, Tmp(GPRInfo::regT3), Arg::addr(base, 3 * sizeof(int64_t)));
1072     root->append(Move, nullptr, Tmp(GPRInfo::regT4), Arg::addr(base, 4 * sizeof(int64_t)));
1073     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1074     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1075
1076     memset(things, 0, sizeof(things));
1077     
1078     CHECK(!compileAndRun<int>(proc));
1079
1080     CHECK(things[0] == 10000000000000000ll);
1081     CHECK(things[1] == 20000000000000000ll);
1082     CHECK(things[2] == 30000000000000000ll);
1083     CHECK(things[3] == 30000000000000000ll);
1084     CHECK(things[4] == static_cast<uint32_t>(40000000000000000ll));
1085 }
1086
1087 void testShuffleShiftMemory()
1088 {
1089     B3::Procedure proc;
1090     Code& code = proc.code();
1091
1092     int32_t memory[2];
1093     memory[0] = 35;
1094     memory[1] = 36;
1095
1096     BasicBlock* root = code.addBlock();
1097     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1098     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1099     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1100     root->append(
1101         Shuffle, nullptr,
1102         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1103         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1104         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Arg::widthArg(Width32));
1105
1106     int32_t things[2];
1107     Tmp base = code.newTmp(GP);
1108     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1109     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1110     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1111     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1112     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1113
1114     memset(things, 0, sizeof(things));
1115     
1116     CHECK(!compileAndRun<int>(proc));
1117
1118     CHECK(things[0] == 1);
1119     CHECK(things[1] == 1);
1120     CHECK(memory[0] == 35);
1121     CHECK(memory[1] == 35);
1122 }
1123
1124 void testShuffleShiftMemoryLong()
1125 {
1126     B3::Procedure proc;
1127     Code& code = proc.code();
1128
1129     int32_t memory[2];
1130     memory[0] = 35;
1131     memory[1] = 36;
1132
1133     BasicBlock* root = code.addBlock();
1134     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1135     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1136     loadConstant(root, 3, Tmp(GPRInfo::regT2));
1137     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT3));
1138     root->append(
1139         Shuffle, nullptr,
1140         
1141         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1142         
1143         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT3), 0 * sizeof(int32_t)),
1144         Arg::widthArg(Width32),
1145         
1146         Arg::addr(Tmp(GPRInfo::regT3), 0 * sizeof(int32_t)),
1147         Arg::addr(Tmp(GPRInfo::regT3), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1148
1149         Arg::addr(Tmp(GPRInfo::regT3), 1 * sizeof(int32_t)), Tmp(GPRInfo::regT2),
1150         Arg::widthArg(Width32));
1151
1152     int32_t things[3];
1153     Tmp base = code.newTmp(GP);
1154     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1155     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1156     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1157     root->append(Move32, nullptr, Tmp(GPRInfo::regT2), Arg::addr(base, 2 * sizeof(int32_t)));
1158     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1159     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1160
1161     memset(things, 0, sizeof(things));
1162     
1163     CHECK(!compileAndRun<int>(proc));
1164
1165     CHECK(things[0] == 1);
1166     CHECK(things[1] == 1);
1167     CHECK(things[2] == 36);
1168     CHECK(memory[0] == 2);
1169     CHECK(memory[1] == 35);
1170 }
1171
1172 void testShuffleShiftMemoryAllRegs()
1173 {
1174     B3::Procedure proc;
1175     Code& code = proc.code();
1176
1177     int32_t memory[2];
1178     memory[0] = 35;
1179     memory[1] = 36;
1180
1181     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1182     regs.removeFirst(Reg(GPRInfo::regT0));
1183
1184     BasicBlock* root = code.addBlock();
1185     for (unsigned i = 0; i < regs.size(); ++i)
1186         loadConstant(root, i + 1, Tmp(regs[i]));
1187     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1188     Inst& shuffle = root->append(
1189         Shuffle, nullptr,
1190         
1191         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int32_t)),
1192         Arg::widthArg(Width32),
1193         
1194         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int32_t)),
1195         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1196
1197         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int32_t)), Tmp(regs[1]),
1198         Arg::widthArg(Width32));
1199
1200     for (unsigned i = 2; i < regs.size(); ++i)
1201         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width32));
1202
1203     Vector<int32_t> things(regs.size(), 666);
1204     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1205     for (unsigned i = 0; i < regs.size(); ++i) {
1206         root->append(
1207             Move32, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int32_t)));
1208     }
1209     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1210     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1211
1212     CHECK(!compileAndRun<int>(proc));
1213
1214     CHECK(things[0] == 1);
1215     CHECK(things[1] == 36);
1216     for (unsigned i = 2; i < regs.size(); ++i)
1217         CHECK(things[i] == static_cast<int32_t>(i));
1218     CHECK(memory[0] == 1);
1219     CHECK(memory[1] == 35);
1220 }
1221
1222 void testShuffleShiftMemoryAllRegs64()
1223 {
1224     B3::Procedure proc;
1225     Code& code = proc.code();
1226
1227     int64_t memory[2];
1228     memory[0] = 35000000000000ll;
1229     memory[1] = 36000000000000ll;
1230
1231     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1232     regs.removeFirst(Reg(GPRInfo::regT0));
1233
1234     BasicBlock* root = code.addBlock();
1235     for (unsigned i = 0; i < regs.size(); ++i)
1236         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1237     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1238     Inst& shuffle = root->append(
1239         Shuffle, nullptr,
1240         
1241         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1242         Arg::widthArg(Width64),
1243         
1244         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1245         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1246
1247         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1248         Arg::widthArg(Width64));
1249
1250     for (unsigned i = 2; i < regs.size(); ++i)
1251         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1252
1253     Vector<int64_t> things(regs.size(), 666);
1254     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1255     for (unsigned i = 0; i < regs.size(); ++i) {
1256         root->append(
1257             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1258     }
1259     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1260     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1261
1262     CHECK(!compileAndRun<int>(proc));
1263
1264     CHECK(things[0] == 1000000000000ll);
1265     CHECK(things[1] == 36000000000000ll);
1266     for (unsigned i = 2; i < regs.size(); ++i)
1267         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1268     CHECK(memory[0] == 1000000000000ll);
1269     CHECK(memory[1] == 35000000000000ll);
1270 }
1271
1272 int64_t combineHiLo(int64_t high, int64_t low)
1273 {
1274     union {
1275         int64_t value;
1276         int32_t halves[2];
1277     } u;
1278     u.value = high;
1279     u.halves[0] = static_cast<int32_t>(low);
1280     return u.value;
1281 }
1282
1283 void testShuffleShiftMemoryAllRegsMixedWidth()
1284 {
1285     B3::Procedure proc;
1286     Code& code = proc.code();
1287
1288     int64_t memory[2];
1289     memory[0] = 35000000000000ll;
1290     memory[1] = 36000000000000ll;
1291
1292     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1293     regs.removeFirst(Reg(GPRInfo::regT0));
1294
1295     BasicBlock* root = code.addBlock();
1296     for (unsigned i = 0; i < regs.size(); ++i)
1297         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1298     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1299     Inst& shuffle = root->append(
1300         Shuffle, nullptr,
1301         
1302         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1303         Arg::widthArg(Width32),
1304         
1305         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1306         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1307
1308         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1309         Arg::widthArg(Width32));
1310
1311     for (unsigned i = 2; i < regs.size(); ++i) {
1312         shuffle.append(
1313             Tmp(regs[i - 1]), Tmp(regs[i]),
1314             (i & 1) ? Arg::widthArg(Width32) : Arg::widthArg(Width64));
1315     }
1316
1317     Vector<int64_t> things(regs.size(), 666);
1318     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1319     for (unsigned i = 0; i < regs.size(); ++i) {
1320         root->append(
1321             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1322     }
1323     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1324     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1325
1326     CHECK(!compileAndRun<int>(proc));
1327
1328     CHECK(things[0] == 1000000000000ll);
1329     CHECK(things[1] == static_cast<uint32_t>(36000000000000ll));
1330     for (unsigned i = 2; i < regs.size(); ++i) {
1331         int64_t value = static_cast<int64_t>(i) * 1000000000000ll;
1332         CHECK(things[i] == ((i & 1) ? static_cast<uint32_t>(value) : value));
1333     }
1334     CHECK(memory[0] == combineHiLo(35000000000000ll, 1000000000000ll));
1335     CHECK(memory[1] == 35000000000000ll);
1336 }
1337
1338 void testShuffleRotateMemory()
1339 {
1340     B3::Procedure proc;
1341     Code& code = proc.code();
1342
1343     int32_t memory[2];
1344     memory[0] = 35;
1345     memory[1] = 36;
1346
1347     BasicBlock* root = code.addBlock();
1348     loadConstant(root, 1, Tmp(GPRInfo::regT0));
1349     loadConstant(root, 2, Tmp(GPRInfo::regT1));
1350     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1351     root->append(
1352         Shuffle, nullptr,
1353         
1354         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1355
1356         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1357         Arg::widthArg(Width32),
1358         
1359         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int32_t)),
1360         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Arg::widthArg(Width32),
1361
1362         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int32_t)), Tmp(GPRInfo::regT0),
1363         Arg::widthArg(Width32));
1364
1365     int32_t things[2];
1366     Tmp base = code.newTmp(GP);
1367     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1368     root->append(Move32, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int32_t)));
1369     root->append(Move32, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int32_t)));
1370     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1371     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1372
1373     memset(things, 0, sizeof(things));
1374     
1375     CHECK(!compileAndRun<int>(proc));
1376
1377     CHECK(things[0] == 36);
1378     CHECK(things[1] == 1);
1379     CHECK(memory[0] == 2);
1380     CHECK(memory[1] == 35);
1381 }
1382
1383 void testShuffleRotateMemory64()
1384 {
1385     B3::Procedure proc;
1386     Code& code = proc.code();
1387
1388     int64_t memory[2];
1389     memory[0] = 35000000000000ll;
1390     memory[1] = 36000000000000ll;
1391
1392     BasicBlock* root = code.addBlock();
1393     loadConstant(root, 1000000000000ll, Tmp(GPRInfo::regT0));
1394     loadConstant(root, 2000000000000ll, Tmp(GPRInfo::regT1));
1395     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1396     root->append(
1397         Shuffle, nullptr,
1398         
1399         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width64),
1400
1401         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1402         Arg::widthArg(Width64),
1403         
1404         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1405         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1406
1407         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Tmp(GPRInfo::regT0),
1408         Arg::widthArg(Width64));
1409
1410     int64_t things[2];
1411     Tmp base = code.newTmp(GP);
1412     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1413     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1414     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1415     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1416     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1417
1418     memset(things, 0, sizeof(things));
1419     
1420     CHECK(!compileAndRun<int>(proc));
1421
1422     CHECK(things[0] == 36000000000000ll);
1423     CHECK(things[1] == 1000000000000ll);
1424     CHECK(memory[0] == 2000000000000ll);
1425     CHECK(memory[1] == 35000000000000ll);
1426 }
1427
1428 void testShuffleRotateMemoryMixedWidth()
1429 {
1430     B3::Procedure proc;
1431     Code& code = proc.code();
1432
1433     int64_t memory[2];
1434     memory[0] = 35000000000000ll;
1435     memory[1] = 36000000000000ll;
1436
1437     BasicBlock* root = code.addBlock();
1438     loadConstant(root, 1000000000000ll, Tmp(GPRInfo::regT0));
1439     loadConstant(root, 2000000000000ll, Tmp(GPRInfo::regT1));
1440     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT2));
1441     root->append(
1442         Shuffle, nullptr,
1443         
1444         Tmp(GPRInfo::regT0), Tmp(GPRInfo::regT1), Arg::widthArg(Width32),
1445
1446         Tmp(GPRInfo::regT1), Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1447         Arg::widthArg(Width64),
1448         
1449         Arg::addr(Tmp(GPRInfo::regT2), 0 * sizeof(int64_t)),
1450         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Arg::widthArg(Width32),
1451
1452         Arg::addr(Tmp(GPRInfo::regT2), 1 * sizeof(int64_t)), Tmp(GPRInfo::regT0),
1453         Arg::widthArg(Width64));
1454
1455     int64_t things[2];
1456     Tmp base = code.newTmp(GP);
1457     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1458     root->append(Move, nullptr, Tmp(GPRInfo::regT0), Arg::addr(base, 0 * sizeof(int64_t)));
1459     root->append(Move, nullptr, Tmp(GPRInfo::regT1), Arg::addr(base, 1 * sizeof(int64_t)));
1460     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1461     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1462
1463     memset(things, 0, sizeof(things));
1464     
1465     CHECK(!compileAndRun<int>(proc));
1466
1467     CHECK(things[0] == 36000000000000ll);
1468     CHECK(things[1] == static_cast<uint32_t>(1000000000000ll));
1469     CHECK(memory[0] == 2000000000000ll);
1470     CHECK(memory[1] == combineHiLo(36000000000000ll, 35000000000000ll));
1471 }
1472
1473 void testShuffleRotateMemoryAllRegs64()
1474 {
1475     B3::Procedure proc;
1476     Code& code = proc.code();
1477
1478     int64_t memory[2];
1479     memory[0] = 35000000000000ll;
1480     memory[1] = 36000000000000ll;
1481
1482     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1483     regs.removeFirst(Reg(GPRInfo::regT0));
1484
1485     BasicBlock* root = code.addBlock();
1486     for (unsigned i = 0; i < regs.size(); ++i)
1487         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1488     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1489     Inst& shuffle = root->append(
1490         Shuffle, nullptr,
1491         
1492         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1493         Arg::widthArg(Width64),
1494         
1495         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1496         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1497
1498         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1499         Arg::widthArg(Width64),
1500
1501         regs.last(), regs[0], Arg::widthArg(Width64));
1502
1503     for (unsigned i = 2; i < regs.size(); ++i)
1504         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1505
1506     Vector<int64_t> things(regs.size(), 666);
1507     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1508     for (unsigned i = 0; i < regs.size(); ++i) {
1509         root->append(
1510             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1511     }
1512     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1513     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1514
1515     CHECK(!compileAndRun<int>(proc));
1516
1517     CHECK(things[0] == static_cast<int64_t>(regs.size()) * 1000000000000ll);
1518     CHECK(things[1] == 36000000000000ll);
1519     for (unsigned i = 2; i < regs.size(); ++i)
1520         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1521     CHECK(memory[0] == 1000000000000ll);
1522     CHECK(memory[1] == 35000000000000ll);
1523 }
1524
1525 void testShuffleRotateMemoryAllRegsMixedWidth()
1526 {
1527     B3::Procedure proc;
1528     Code& code = proc.code();
1529
1530     int64_t memory[2];
1531     memory[0] = 35000000000000ll;
1532     memory[1] = 36000000000000ll;
1533
1534     Vector<Reg> regs = code.regsInPriorityOrder(GP);
1535     regs.removeFirst(Reg(GPRInfo::regT0));
1536
1537     BasicBlock* root = code.addBlock();
1538     for (unsigned i = 0; i < regs.size(); ++i)
1539         loadConstant(root, (i + 1) * 1000000000000ll, Tmp(regs[i]));
1540     root->append(Move, nullptr, Arg::immPtr(&memory), Tmp(GPRInfo::regT0));
1541     Inst& shuffle = root->append(
1542         Shuffle, nullptr,
1543         
1544         Tmp(regs[0]), Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1545         Arg::widthArg(Width32),
1546         
1547         Arg::addr(Tmp(GPRInfo::regT0), 0 * sizeof(int64_t)),
1548         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Arg::widthArg(Width64),
1549
1550         Arg::addr(Tmp(GPRInfo::regT0), 1 * sizeof(int64_t)), Tmp(regs[1]),
1551         Arg::widthArg(Width32),
1552
1553         regs.last(), regs[0], Arg::widthArg(Width32));
1554
1555     for (unsigned i = 2; i < regs.size(); ++i)
1556         shuffle.append(Tmp(regs[i - 1]), Tmp(regs[i]), Arg::widthArg(Width64));
1557
1558     Vector<int64_t> things(regs.size(), 666);
1559     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things[0])), Tmp(GPRInfo::regT0));
1560     for (unsigned i = 0; i < regs.size(); ++i) {
1561         root->append(
1562             Move, nullptr, Tmp(regs[i]), Arg::addr(Tmp(GPRInfo::regT0), i * sizeof(int64_t)));
1563     }
1564     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1565     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1566
1567     CHECK(!compileAndRun<int>(proc));
1568
1569     CHECK(things[0] == static_cast<uint32_t>(static_cast<int64_t>(regs.size()) * 1000000000000ll));
1570     CHECK(things[1] == static_cast<uint32_t>(36000000000000ll));
1571     for (unsigned i = 2; i < regs.size(); ++i)
1572         CHECK(things[i] == static_cast<int64_t>(i) * 1000000000000ll);
1573     CHECK(memory[0] == combineHiLo(35000000000000ll, 1000000000000ll));
1574     CHECK(memory[1] == 35000000000000ll);
1575 }
1576
1577 void testShuffleSwapDouble()
1578 {
1579     B3::Procedure proc;
1580     Code& code = proc.code();
1581
1582     BasicBlock* root = code.addBlock();
1583     loadDoubleConstant(root, 1, Tmp(FPRInfo::fpRegT0), Tmp(GPRInfo::regT0));
1584     loadDoubleConstant(root, 2, Tmp(FPRInfo::fpRegT1), Tmp(GPRInfo::regT0));
1585     loadDoubleConstant(root, 3, Tmp(FPRInfo::fpRegT2), Tmp(GPRInfo::regT0));
1586     loadDoubleConstant(root, 4, Tmp(FPRInfo::fpRegT3), Tmp(GPRInfo::regT0));
1587     root->append(
1588         Shuffle, nullptr,
1589         Tmp(FPRInfo::fpRegT2), Tmp(FPRInfo::fpRegT3), Arg::widthArg(Width64),
1590         Tmp(FPRInfo::fpRegT3), Tmp(FPRInfo::fpRegT2), Arg::widthArg(Width64));
1591
1592     double things[4];
1593     Tmp base = code.newTmp(GP);
1594     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1595     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT0), Arg::addr(base, 0 * sizeof(double)));
1596     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT1), Arg::addr(base, 1 * sizeof(double)));
1597     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT2), Arg::addr(base, 2 * sizeof(double)));
1598     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT3), Arg::addr(base, 3 * sizeof(double)));
1599     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1600     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1601
1602     memset(things, 0, sizeof(things));
1603     
1604     CHECK(!compileAndRun<int>(proc));
1605
1606     CHECK(things[0] == 1);
1607     CHECK(things[1] == 2);
1608     CHECK(things[2] == 4);
1609     CHECK(things[3] == 3);
1610 }
1611
1612 void testShuffleShiftDouble()
1613 {
1614     B3::Procedure proc;
1615     Code& code = proc.code();
1616
1617     BasicBlock* root = code.addBlock();
1618     loadDoubleConstant(root, 1, Tmp(FPRInfo::fpRegT0), Tmp(GPRInfo::regT0));
1619     loadDoubleConstant(root, 2, Tmp(FPRInfo::fpRegT1), Tmp(GPRInfo::regT0));
1620     loadDoubleConstant(root, 3, Tmp(FPRInfo::fpRegT2), Tmp(GPRInfo::regT0));
1621     loadDoubleConstant(root, 4, Tmp(FPRInfo::fpRegT3), Tmp(GPRInfo::regT0));
1622     root->append(
1623         Shuffle, nullptr,
1624         Tmp(FPRInfo::fpRegT2), Tmp(FPRInfo::fpRegT3), Arg::widthArg(Width64));
1625
1626     double things[4];
1627     Tmp base = code.newTmp(GP);
1628     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1629     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT0), Arg::addr(base, 0 * sizeof(double)));
1630     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT1), Arg::addr(base, 1 * sizeof(double)));
1631     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT2), Arg::addr(base, 2 * sizeof(double)));
1632     root->append(MoveDouble, nullptr, Tmp(FPRInfo::fpRegT3), Arg::addr(base, 3 * sizeof(double)));
1633     root->append(Move, nullptr, Arg::imm(0), Tmp(GPRInfo::returnValueGPR));
1634     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1635
1636     memset(things, 0, sizeof(things));
1637     
1638     CHECK(!compileAndRun<int>(proc));
1639
1640     CHECK(things[0] == 1);
1641     CHECK(things[1] == 2);
1642     CHECK(things[2] == 3);
1643     CHECK(things[3] == 3);
1644 }
1645
1646 #if CPU(X86) || CPU(X86_64)
1647 void testX86VMULSD()
1648 {
1649     B3::Procedure proc;
1650     Code& code = proc.code();
1651
1652     BasicBlock* root = code.addBlock();
1653     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(FPRInfo::argumentFPR1), Tmp(FPRInfo::argumentFPR2));
1654     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1655     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1656
1657     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1658 }
1659
1660 void testX86VMULSDDestRex()
1661 {
1662     B3::Procedure proc;
1663     Code& code = proc.code();
1664
1665     BasicBlock* root = code.addBlock();
1666     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm15));
1667     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1668     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1669
1670     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1671 }
1672
1673 void testX86VMULSDOp1DestRex()
1674 {
1675     B3::Procedure proc;
1676     Code& code = proc.code();
1677
1678     BasicBlock* root = code.addBlock();
1679     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1680     root->append(MulDouble, nullptr, Tmp(X86Registers::xmm14), Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm15));
1681     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1682     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1683
1684     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1685 }
1686
1687 void testX86VMULSDOp2DestRex()
1688 {
1689     B3::Procedure proc;
1690     Code& code = proc.code();
1691
1692     BasicBlock* root = code.addBlock();
1693     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm14));
1694     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm15));
1695     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1696     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1697
1698     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1699 }
1700
1701 void testX86VMULSDOpsDestRex()
1702 {
1703     B3::Procedure proc;
1704     Code& code = proc.code();
1705
1706     BasicBlock* root = code.addBlock();
1707     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1708     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR1), Tmp(X86Registers::xmm13));
1709     root->append(MulDouble, nullptr, Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm13), Tmp(X86Registers::xmm15));
1710     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1711     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1712
1713     CHECK(compileAndRun<double>(proc, 2.4, 4.2, pureNaN()) == 2.4 * 4.2);
1714 }
1715
1716 void testX86VMULSDAddr()
1717 {
1718     B3::Procedure proc;
1719     Code& code = proc.code();
1720
1721     BasicBlock* root = code.addBlock();
1722     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(GPRInfo::argumentGPR0), - 16), Tmp(FPRInfo::argumentFPR2));
1723     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1724     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1725
1726     double secondArg = 4.2;
1727     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 2, pureNaN()) == 2.4 * 4.2);
1728 }
1729
1730 void testX86VMULSDAddrOpRexAddr()
1731 {
1732     B3::Procedure proc;
1733     Code& code = proc.code();
1734
1735     BasicBlock* root = code.addBlock();
1736     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1737     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(X86Registers::r13), - 16), Tmp(FPRInfo::argumentFPR2));
1738     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR2), Tmp(FPRInfo::returnValueFPR));
1739     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1740
1741     double secondArg = 4.2;
1742     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 2, pureNaN()) == 2.4 * 4.2);
1743 }
1744
1745 void testX86VMULSDDestRexAddr()
1746 {
1747     B3::Procedure proc;
1748     Code& code = proc.code();
1749
1750     BasicBlock* root = code.addBlock();
1751     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(GPRInfo::argumentGPR0), 16), Tmp(X86Registers::xmm15));
1752     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1753     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1754
1755     double secondArg = 4.2;
1756     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 2, pureNaN()) == 2.4 * 4.2);
1757 }
1758
1759 void testX86VMULSDRegOpDestRexAddr()
1760 {
1761     B3::Procedure proc;
1762     Code& code = proc.code();
1763
1764     BasicBlock* root = code.addBlock();
1765     root->append(MoveDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm14));
1766     root->append(MulDouble, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0)), Tmp(X86Registers::xmm14), Tmp(X86Registers::xmm15));
1767     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1768     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1769
1770     double secondArg = 4.2;
1771     CHECK(compileAndRun<double>(proc, 2.4, &secondArg, pureNaN()) == 2.4 * 4.2);
1772 }
1773
1774 void testX86VMULSDAddrOpDestRexAddr()
1775 {
1776     B3::Procedure proc;
1777     Code& code = proc.code();
1778
1779     BasicBlock* root = code.addBlock();
1780     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1781     root->append(MulDouble, nullptr, Tmp(FPRInfo::argumentFPR0), Arg::addr(Tmp(X86Registers::r13), 8), Tmp(X86Registers::xmm15));
1782     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm15), Tmp(FPRInfo::returnValueFPR));
1783     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1784
1785     double secondArg = 4.2;
1786     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 1, pureNaN()) == 2.4 * 4.2);
1787 }
1788
1789 void testX86VMULSDBaseNeedsRex()
1790 {
1791     B3::Procedure proc;
1792     Code& code = proc.code();
1793
1794     BasicBlock* root = code.addBlock();
1795     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13));
1796     root->append(MulDouble, nullptr, Arg::index(Tmp(X86Registers::r13), Tmp(GPRInfo::argumentGPR1)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1797     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1798     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1799
1800     double secondArg = 4.2;
1801     uint64_t index = 8;
1802     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 1, index, pureNaN()) == 2.4 * 4.2);
1803 }
1804
1805 void testX86VMULSDIndexNeedsRex()
1806 {
1807     B3::Procedure proc;
1808     Code& code = proc.code();
1809
1810     BasicBlock* root = code.addBlock();
1811     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR1), Tmp(X86Registers::r13));
1812     root->append(MulDouble, nullptr, Arg::index(Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r13)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1813     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1814     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1815
1816     double secondArg = 4.2;
1817     uint64_t index = - 8;
1818     CHECK(compileAndRun<double>(proc, 2.4, &secondArg + 1, index, pureNaN()) == 2.4 * 4.2);
1819 }
1820
1821 void testX86VMULSDBaseIndexNeedRex()
1822 {
1823     B3::Procedure proc;
1824     Code& code = proc.code();
1825
1826     BasicBlock* root = code.addBlock();
1827     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR0), Tmp(X86Registers::r12));
1828     root->append(Move, nullptr, Tmp(GPRInfo::argumentGPR1), Tmp(X86Registers::r13));
1829     root->append(MulDouble, nullptr, Arg::index(Tmp(X86Registers::r12), Tmp(X86Registers::r13)), Tmp(FPRInfo::argumentFPR0), Tmp(X86Registers::xmm0));
1830     root->append(MoveDouble, nullptr, Tmp(X86Registers::xmm0), Tmp(FPRInfo::returnValueFPR));
1831     root->append(RetDouble, nullptr, Tmp(FPRInfo::returnValueFPR));
1832
1833     double secondArg = 4.2;
1834     uint64_t index = 16;
1835     CHECK(compileAndRun<double>(proc, 2.4, &secondArg - 2, index, pureNaN()) == 2.4 * 4.2);
1836 }
1837 #endif // #if CPU(X86) || CPU(X86_64)
1838
1839 #if CPU(ARM64)
1840 void testInvalidateCachedTempRegisters()
1841 {
1842     B3::Procedure proc;
1843     Code& code = proc.code();
1844     BasicBlock* root = code.addBlock();
1845
1846     int32_t things[4];
1847     things[0] = 0x12000000;
1848     things[1] = 0x340000;
1849     things[2] = 0x5600;
1850     things[3] = 0x78;
1851     Tmp base = code.newTmp(GP);
1852     GPRReg tmp = GPRInfo::regT1;
1853     proc.pinRegister(tmp);
1854
1855     root->append(Move, nullptr, Arg::bigImm(bitwise_cast<intptr_t>(&things)), base);
1856
1857     B3::BasicBlock* patchPoint1Root = proc.addBlock();
1858     B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
1859
1860     // In Patchpoint, Load things[0] -> tmp. This will materialize the address in x17 (dataMemoryRegister).
1861     B3::PatchpointValue* patchpoint1 = patchPoint1Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1862     patchpoint1->clobber(RegisterSet::macroScratchRegisters());
1863     patchpoint1->setGenerator(
1864         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1865             AllowMacroScratchRegisterUsage allowScratch(jit);
1866             jit.load32(&things, tmp);
1867         });
1868     root->append(Patch, patchpoint1, Arg::special(patchpointSpecial));
1869
1870     // Load things[1] -> x17, trashing dataMemoryRegister.
1871     root->append(Move32, nullptr, Arg::addr(base, 1 * sizeof(int32_t)), Tmp(ARM64Registers::x17));
1872     root->append(Add32, nullptr, Tmp(tmp), Tmp(ARM64Registers::x17), Tmp(GPRInfo::returnValueGPR));
1873
1874     // In Patchpoint, Load things[2] -> tmp. This should not reuse the prior contents of x17.
1875     B3::BasicBlock* patchPoint2Root = proc.addBlock();
1876     B3::PatchpointValue* patchpoint2 = patchPoint2Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1877     patchpoint2->clobber(RegisterSet::macroScratchRegisters());
1878     patchpoint2->setGenerator(
1879         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1880             AllowMacroScratchRegisterUsage allowScratch(jit);
1881             jit.load32(&things[2], tmp);
1882         });
1883     root->append(Patch, patchpoint2, Arg::special(patchpointSpecial));
1884
1885     root->append(Add32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR), Tmp(GPRInfo::returnValueGPR));
1886
1887     // In patchpoint, Store 0x78 -> things[3].
1888     // This will use and cache both x16 (dataMemoryRegister) and x17 (dataTempRegister).
1889     B3::BasicBlock* patchPoint3Root = proc.addBlock();
1890     B3::PatchpointValue* patchpoint3 = patchPoint3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1891     patchpoint3->clobber(RegisterSet::macroScratchRegisters());
1892     patchpoint3->setGenerator(
1893         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1894             AllowMacroScratchRegisterUsage allowScratch(jit);
1895             jit.store32(CCallHelpers::TrustedImm32(0x78), &things[3]);
1896         });
1897     root->append(Patch, patchpoint3, Arg::special(patchpointSpecial));
1898
1899     // Set x16 to 0xdead, trashing x16.
1900     root->append(Move, nullptr, Arg::bigImm(0xdead), Tmp(ARM64Registers::x16));
1901     root->append(Xor32, nullptr, Tmp(ARM64Registers::x16), Tmp(GPRInfo::returnValueGPR));
1902
1903     // In patchpoint, again Store 0x78 -> things[3].
1904     // This should rematerialize both x16 (dataMemoryRegister) and x17 (dataTempRegister).
1905     B3::BasicBlock* patchPoint4Root = proc.addBlock();
1906     B3::PatchpointValue* patchpoint4 = patchPoint4Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1907     patchpoint4->clobber(RegisterSet::macroScratchRegisters());
1908     patchpoint4->setGenerator(
1909         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1910             AllowMacroScratchRegisterUsage allowScratch(jit);
1911             jit.store32(CCallHelpers::TrustedImm32(0x78), &things[3]);
1912         });
1913     root->append(Patch, patchpoint4, Arg::special(patchpointSpecial));
1914
1915     root->append(Move, nullptr, Arg::bigImm(0xdead), Tmp(tmp));
1916     root->append(Xor32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR));
1917     root->append(Move32, nullptr, Arg::addr(base, 3 * sizeof(int32_t)), Tmp(tmp));
1918     root->append(Add32, nullptr, Tmp(tmp), Tmp(GPRInfo::returnValueGPR), Tmp(GPRInfo::returnValueGPR));
1919     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1920
1921     int32_t r = compileAndRun<int32_t>(proc);
1922     CHECK(r == 0x12345678);
1923 }
1924 #endif // #if CPU(ARM64)
1925
1926 void testArgumentRegPinned()
1927 {
1928     B3::Procedure proc;
1929     Code& code = proc.code();
1930     GPRReg pinned = GPRInfo::argumentGPR0;
1931     proc.pinRegister(pinned);
1932
1933     B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
1934
1935     B3::BasicBlock* b3Root = proc.addBlock();
1936     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1937     patchpoint->clobber(RegisterSet(pinned));
1938     patchpoint->setGenerator(
1939         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1940             jit.move(CCallHelpers::TrustedImm32(42), pinned);
1941         });
1942
1943     BasicBlock* root = code.addBlock();
1944
1945     Tmp t1 = code.newTmp(GP);
1946     Tmp t2 = code.newTmp(GP);
1947
1948     root->append(Move, nullptr, Tmp(pinned), t1);
1949     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
1950     root->append(Move, nullptr, Tmp(pinned), t2);
1951     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
1952     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1953
1954     int32_t r = compileAndRun<int32_t>(proc, 10);
1955     CHECK(r == 10 + 42);
1956 }
1957
1958 void testArgumentRegPinned2()
1959 {
1960     B3::Procedure proc;
1961     Code& code = proc.code();
1962     GPRReg pinned = GPRInfo::argumentGPR0;
1963     proc.pinRegister(pinned);
1964
1965     B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
1966
1967     B3::BasicBlock* b3Root = proc.addBlock();
1968     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
1969     patchpoint->clobber({ }); 
1970     patchpoint->setGenerator(
1971         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1972             jit.move(CCallHelpers::TrustedImm32(42), pinned);
1973         });
1974
1975     BasicBlock* root = code.addBlock();
1976
1977     Tmp t1 = code.newTmp(GP);
1978     Tmp t2 = code.newTmp(GP);
1979
1980     // Since the patchpoint does not claim to clobber the pinned register,
1981     // the register allocator is allowed to either coalesce the first move,
1982     // the second move, or neither. The allowed results are:
1983     // - No move coalesced: 52
1984     // - The first move is coalesced: 84
1985     // - The second move is coalesced: 52
1986     root->append(Move, nullptr, Tmp(pinned), t1);
1987     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
1988     root->append(Move, nullptr, Tmp(pinned), t2);
1989     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
1990     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
1991
1992     int32_t r = compileAndRun<int32_t>(proc, 10);
1993     CHECK(r == 52 || r == 84);
1994 }
1995
1996 void testArgumentRegPinned3()
1997 {
1998     B3::Procedure proc;
1999     Code& code = proc.code();
2000     GPRReg pinned = GPRInfo::argumentGPR0;
2001     proc.pinRegister(pinned);
2002
2003     B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
2004
2005     B3::BasicBlock* b3Root = proc.addBlock();
2006     B3::PatchpointValue* patchpoint = b3Root->appendNew<B3::PatchpointValue>(proc, B3::Void, B3::Origin());
2007     patchpoint->clobber(RegisterSet(pinned));
2008     patchpoint->setGenerator(
2009         [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2010             jit.move(CCallHelpers::TrustedImm32(42), pinned);
2011         });
2012
2013     BasicBlock* root = code.addBlock();
2014
2015     Tmp t1 = code.newTmp(GP);
2016     Tmp t2 = code.newTmp(GP);
2017     Tmp t3 = code.newTmp(GP);
2018
2019     root->append(Move, nullptr, Tmp(pinned), t1);
2020     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
2021     root->append(Move, nullptr, Tmp(pinned), t2);
2022     root->append(Patch, patchpoint, Arg::special(patchpointSpecial));
2023     root->append(Move, nullptr, Tmp(pinned), t3);
2024     root->append(Add32, nullptr, t1, t2, Tmp(GPRInfo::returnValueGPR));
2025     root->append(Add32, nullptr, Tmp(GPRInfo::returnValueGPR), t3, Tmp(GPRInfo::returnValueGPR));
2026     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
2027
2028     int32_t r = compileAndRun<int32_t>(proc, 10);
2029     CHECK(r == 10 + 42 + 42);
2030 }
2031
2032 void testLea64()
2033 {
2034     B3::Procedure proc;
2035     Code& code = proc.code();
2036
2037     BasicBlock* root = code.addBlock();
2038
2039     int64_t a = 0x11223344;
2040     int64_t b = 1 << 13;
2041
2042     root->append(Lea64, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0), b), Tmp(GPRInfo::returnValueGPR));
2043     root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));
2044
2045     int64_t r = compileAndRun<int64_t>(proc, a);
2046     CHECK(r == a + b);
2047 }
2048
2049 void testLea32()
2050 {
2051     B3::Procedure proc;
2052     Code& code = proc.code();
2053
2054     BasicBlock* root = code.addBlock();
2055
2056     int32_t a = 0x11223344;
2057     int32_t b = 1 << 13;
2058
2059     root->append(Lea32, nullptr, Arg::addr(Tmp(GPRInfo::argumentGPR0), b), Tmp(GPRInfo::returnValueGPR));
2060     root->append(Ret32, nullptr, Tmp(GPRInfo::returnValueGPR));
2061
2062     int32_t r = compileAndRun<int32_t>(proc, a);
2063     CHECK(r == a + b);
2064 }
2065
2066 #define PREFIX "O", Options::defaultB3OptLevel(), ": "
2067
2068 #define RUN(test) do {                                 \
2069         if (!shouldRun(#test))                         \
2070             break;                                     \
2071         tasks.append(                                  \
2072             createSharedTask<void()>(                  \
2073                 [&] () {                               \
2074                     dataLog(PREFIX #test "...\n");     \
2075                     test;                              \
2076                     dataLog(PREFIX #test ": OK!\n");   \
2077                 }));                                   \
2078     } while (false);
2079
2080 void run(const char* filter)
2081 {
2082     Deque<RefPtr<SharedTask<void()>>> tasks;
2083
2084     auto shouldRun = [&] (const char* testName) -> bool {
2085         return !filter || WTF::findIgnoringASCIICaseWithoutLength(testName, filter) != WTF::notFound;
2086     };
2087
2088     RUN(testSimple());
2089     
2090     RUN(testShuffleSimpleSwap());
2091     RUN(testShuffleSimpleShift());
2092     RUN(testShuffleLongShift());
2093     RUN(testShuffleLongShiftBackwards());
2094     RUN(testShuffleSimpleRotate());
2095     RUN(testShuffleSimpleBroadcast());
2096     RUN(testShuffleBroadcastAllRegs());
2097     RUN(testShuffleTreeShift());
2098     RUN(testShuffleTreeShiftBackward());
2099     RUN(testShuffleTreeShiftOtherBackward());
2100     RUN(testShuffleMultipleShifts());
2101     RUN(testShuffleRotateWithFringe());
2102     RUN(testShuffleRotateWithFringeInWeirdOrder());
2103     RUN(testShuffleRotateWithLongFringe());
2104     RUN(testShuffleMultipleRotates());
2105     RUN(testShuffleShiftAndRotate());
2106     RUN(testShuffleShiftAllRegs());
2107     RUN(testShuffleRotateAllRegs());
2108     RUN(testShuffleSimpleSwap64());
2109     RUN(testShuffleSimpleShift64());
2110     RUN(testShuffleSwapMixedWidth());
2111     RUN(testShuffleShiftMixedWidth());
2112     RUN(testShuffleShiftMemory());
2113     RUN(testShuffleShiftMemoryLong());
2114     RUN(testShuffleShiftMemoryAllRegs());
2115     RUN(testShuffleShiftMemoryAllRegs64());
2116     RUN(testShuffleShiftMemoryAllRegsMixedWidth());
2117     RUN(testShuffleRotateMemory());
2118     RUN(testShuffleRotateMemory64());
2119     RUN(testShuffleRotateMemoryMixedWidth());
2120     RUN(testShuffleRotateMemoryAllRegs64());
2121     RUN(testShuffleRotateMemoryAllRegsMixedWidth());
2122     RUN(testShuffleSwapDouble());
2123     RUN(testShuffleShiftDouble());
2124
2125 #if CPU(X86) || CPU(X86_64)
2126     RUN(testX86VMULSD());
2127     RUN(testX86VMULSDDestRex());
2128     RUN(testX86VMULSDOp1DestRex());
2129     RUN(testX86VMULSDOp2DestRex());
2130     RUN(testX86VMULSDOpsDestRex());
2131
2132     RUN(testX86VMULSDAddr());
2133     RUN(testX86VMULSDAddrOpRexAddr());
2134     RUN(testX86VMULSDDestRexAddr());
2135     RUN(testX86VMULSDRegOpDestRexAddr());
2136     RUN(testX86VMULSDAddrOpDestRexAddr());
2137
2138     RUN(testX86VMULSDBaseNeedsRex());
2139     RUN(testX86VMULSDIndexNeedsRex());
2140     RUN(testX86VMULSDBaseIndexNeedRex());
2141 #endif
2142
2143 #if CPU(ARM64)
2144     RUN(testInvalidateCachedTempRegisters());
2145 #endif
2146
2147     RUN(testArgumentRegPinned());
2148     RUN(testArgumentRegPinned2());
2149     RUN(testArgumentRegPinned3());
2150
2151     RUN(testLea32());
2152     RUN(testLea64());
2153
2154     if (tasks.isEmpty())
2155         usage();
2156
2157     Lock lock;
2158
2159     Vector<Ref<Thread>> threads;
2160     for (unsigned i = filter ? 1 : WTF::numberOfProcessorCores(); i--;) {
2161         threads.append(
2162             Thread::create(
2163                 "testair thread",
2164                 [&] () {
2165                     for (;;) {
2166                         RefPtr<SharedTask<void()>> task;
2167                         {
2168                             LockHolder locker(lock);
2169                             if (tasks.isEmpty())
2170                                 return;
2171                             task = tasks.takeFirst();
2172                         }
2173
2174                         task->run();
2175                     }
2176                 }));
2177     }
2178
2179     for (auto& thread : threads)
2180         thread->waitForCompletion();
2181     crashLock.lock();
2182     crashLock.unlock();
2183 }
2184
2185 } // anonymous namespace
2186
2187 #else // ENABLE(B3_JIT)
2188
2189 static void run(const char*)
2190 {
2191     dataLog("B3 JIT is not enabled.\n");
2192 }
2193
2194 #endif // ENABLE(B3_JIT)
2195
2196 int main(int argc, char** argv)
2197 {
2198     const char* filter = nullptr;
2199     switch (argc) {
2200     case 1:
2201         break;
2202     case 2:
2203         filter = argv[1];
2204         break;
2205     default:
2206         usage();
2207         break;
2208     }
2209     
2210     JSC::initializeThreading();
2211
2212     for (unsigned i = 0; i <= 2; ++i) {
2213         JSC::Options::defaultB3OptLevel() = i;
2214         run(filter);
2215     }
2216
2217     return 0;
2218 }
2219
2220 #if OS(WINDOWS)
2221 extern "C" __declspec(dllexport) int WINAPI dllLauncherEntryPoint(int argc, const char* argv[])
2222 {
2223     return main(argc, const_cast<char**>(argv));
2224 }
2225 #endif