Make JetStream 2
[WebKit-https.git] / PerformanceTests / JetStream2 / JetStreamDriver.js
1 "use strict";
2
3 /*
4  * Copyright (C) 2018 Apple Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
19  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
25  * THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 const preloadResources = !isInBrowser;
29 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
30
31 if (typeof RAMification === "undefined")
32     var RAMification = false;
33
34 if (typeof testIterationCount === "undefined")
35     var testIterationCount = undefined;
36
37 // Used for the promise representing the current benchmark run.
38 this.currentResolve = null;
39 this.currentReject = null;
40
41 const defaultIterationCount = 120;
42 const defaultWorstCaseCount = 4;
43
44 function assert(b, m = "") {
45     if (!b)
46         throw new Error("Bad assertion: " + m);
47 }
48
49 function firstID(benchmark) {
50     return `results-cell-${benchmark.name}-first`;
51 }
52
53 function worst4ID(benchmark) {
54     return `results-cell-${benchmark.name}-worst4`;
55 }
56
57 function avgID(benchmark) {
58     return `results-cell-${benchmark.name}-avg`;
59 }
60
61 function scoreID(benchmark) {
62     return `results-cell-${benchmark.name}-score`;
63 }
64
65 function mean(values) {
66     assert(values instanceof Array);
67     let sum = 0;
68     for (let x of values)
69         sum += x;
70     return sum / values.length;
71 }
72
73 function geomean(values) {
74     assert(values instanceof Array);
75     let product = 1;
76     for (let x of values)
77         product *= x;
78     return product ** (1 / values.length);
79 }
80
81 function toScore(timeValue) {
82     return 5000 / timeValue;
83 }
84
85 function updateUI() {
86     return new Promise((resolve) => { 
87         if (isInBrowser)
88             requestAnimationFrame(() => setTimeout(resolve, 0));
89         else
90             resolve();
91     });
92 }
93
94 function uiFriendlyNumber(num) {
95     if (Number.isInteger(num))
96         return num;
97     return num.toFixed(3);
98 }
99
100 function uiFriendlyDuration(time)
101 {
102     let minutes = time.getMinutes();
103     let seconds = time.getSeconds();
104     let milliSeconds = time.getMilliseconds();
105     let result = "" + minutes + ":";
106
107     result = result + (seconds < 10 ? "0" : "") + seconds + ".";
108     result = result + (milliSeconds < 10 ? "00" : (milliSeconds < 100 ? "0" : "")) + milliSeconds;
109
110     return result;
111 }
112
113 const fileLoader = (function() {
114     class Loader {
115         constructor() {
116             this.requests = new Map;
117         }
118
119         async _loadInternal(url) {
120             if (!isInBrowser)
121                 return Promise.resolve(readFile(url));
122
123             let fetchResponse = await fetch(new Request(url));
124             if (url.indexOf(".js") !== -1)
125                 return await fetchResponse.text();
126             else if (url.indexOf(".wasm") !== -1)
127                 return await fetchResponse.arrayBuffer();
128
129             throw new Error("should not be reached!");
130         }
131
132         async load(url) {
133             if (this.requests.has(url))
134                 return (await this.requests.get(url));
135
136             let promise = this._loadInternal(url);
137             this.requests.set(url, promise);
138             return (await promise);
139         }
140     }
141     return new Loader;
142 })();
143
144 class Driver {
145     constructor() {
146         this.benchmarks = [];
147     }
148
149     addPlan(plan, BenchmarkClass = DefaultBenchmark) {
150         this.benchmarks.push(new BenchmarkClass(plan));
151     }
152
153     async start() {
154         if (isInBrowser)
155             document.getElementById("status").innerHTML = `Running...`;
156         else
157             console.log("Starting JetStream2");
158
159         await updateUI();
160
161         let start = Date.now();
162         for (let benchmark of this.benchmarks) {
163             benchmark.updateUIBeforeRun();
164
165             await updateUI();
166
167             try {
168
169                 await benchmark.run();
170             } catch(e) {
171                 JetStream.reportError(benchmark);
172                 throw e;
173             }
174
175             benchmark.updateUIAfterRun();
176         }
177         let totalTime = Date.now() - start;
178         if (measureTotalTimeAsSubtest) {
179             if (isInBrowser)
180                 document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime);
181             else
182                 console.log("Total time:", uiFriendlyNumber(totalTime));
183             allScores.push(totalTime);
184         }
185
186         let allScores = [];
187         for (let benchmark of this.benchmarks)
188             allScores.push(benchmark.score);
189
190         if (isInBrowser) {
191             document.getElementById("result-summary").innerHTML = "<label>Score</label><br><span class=\"score\">" + uiFriendlyNumber(geomean(allScores)) + "</span>";
192             document.getElementById("status").innerHTML = `Done`;
193         } else
194             console.log("\nTotal Score: ", uiFriendlyNumber(geomean(allScores)), "\n");
195
196         this.reportScoreToRunBenchmarkRunner();
197     }
198
199     runCode(string)
200     {
201         if (!isInBrowser) {
202             let scripts = string;
203             let globalObject = runString("");
204             globalObject.console = {log:globalObject.print}
205             globalObject.top = {
206                 currentResolve,
207                 currentReject
208             };
209             for (let script of scripts)
210                 globalObject.loadString(script);
211             return globalObject;
212         }
213
214         var magic = document.getElementById("magic");
215         magic.contentDocument.body.textContent = "";
216         magic.contentDocument.body.innerHTML = "<iframe id=\"magicframe\" frameborder=\"0\">";
217
218         var magicFrame = magic.contentDocument.getElementById("magicframe");
219         magicFrame.contentDocument.open();
220         magicFrame.contentDocument.write("<!DOCTYPE html><head><title>benchmark payload</title></head><body>\n" + string + "</body></html>");
221
222         return magicFrame;
223     }
224
225     prepareToRun()
226     {
227         this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
228
229         let groups = new Map;
230         let names = new Set;
231         for (let benchmark of this.benchmarks) {
232             names.add(benchmark.name);
233             let identifier = JSON.stringify(benchmark.constructor.scoreDescription());
234             if (!groups.has(identifier))
235                 groups.set(identifier, []);
236             groups.get(identifier).push(benchmark);
237         }
238
239         if (names.size !== this.benchmarks.length)
240             throw new Error("Names of benchmarks must be unique");
241
242         let text = "";
243         let newBenchmarks = [];
244         for (let [id, benchmarks] of groups) {
245             let description = JSON.parse(id);
246             if (isInBrowser) {
247                 text += `<tr> <th> Benchmark </th>`;
248                 for (let score of description)
249                     text += `<th> ${score} </th>`;
250             }
251             for (let benchmark of benchmarks) {
252                 newBenchmarks.push(benchmark);
253
254                 if (isInBrowser) {
255                     text +=
256                         `<tr id="row-for-${benchmark.name}">
257                         <!-- FIXME: link to benchmark explanation -->
258                         <td class="benchmark-name"> ${benchmark.name} </td>`;
259                     for (let id of benchmark.scoreIdentifiers())
260                         text += `<td class="result" id="${id}">&mdash;</td>`
261                     text += `</tr>`;
262                 }
263             }
264         }
265
266         if (measureTotalTimeAsSubtest) {
267             if (isInBrowser) {
268                 text += 
269                     `<tr>
270                         <td class="benchmark-name"> Total time </td>
271                         <td class="result" id="benchmark-total-time-score">&mdash;</td>
272                     </tr>`;
273             }
274         }
275
276         if (isInBrowser) {
277             let resultsTable = document.getElementById("results");
278             resultsTable.innerHTML = text;
279
280             document.getElementById("magic").textContent = "";
281         }
282     }
283
284     reportError(benchmark)
285     {
286         for (let id of benchmark.scoreIdentifiers())
287             document.getElementById(id).innerHTML = "error";
288     }
289
290     async initialize() {
291         await this.fetchResources();
292         this.prepareToRun();
293         if (isInBrowser && window.location.search == '?report=true') {
294             setTimeout(() => this.start(), 1000);
295         }
296     }
297
298     async fetchResources() {
299         for (let benchmark of this.benchmarks)
300             await benchmark.fetchResources();
301
302         if (!isInBrowser)
303             return;
304
305         let status = document.getElementById("status");
306         status.innerHTML = `<a href="javascript:JetStream.start()">Start Test</a>`;
307         status.onclick = () => {
308             status.onclick = null;
309             JetStream.start();
310             return false;
311         }
312     }
313
314     async reportScoreToRunBenchmarkRunner()
315     {
316         if (!isInBrowser)
317             return;
318
319         if (window.location.search !== '?report=true')
320             return;
321
322         let results = {};
323         for (let benchmark of this.benchmarks) {
324             const subResults = {}
325             const subTimes = benchmark.subTimes();
326             for (const name in subTimes) {
327                 subResults[name] = {"metrics": {"Time": {"current": [subTimes[name]]}}};
328             }
329             results[benchmark.name] = {
330                 "metrics" : {
331                     "Score" : {"current" : [benchmark.score]},
332                     "Time": ["Geometric"],
333                 },
334                 "tests": subResults,
335             };;
336         }
337
338         results = {"JetStream2.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
339
340         const content = JSON.stringify(results);
341         await fetch("/report", {
342             method: "POST",
343             heeaders: {
344                 "Content-Type": "application/json",
345                 "Content-Length": content.length,
346                 "Connection": "close",
347             },
348             body: content,
349         });
350     }
351 };
352
353 class Benchmark {
354     constructor(plan) 
355     {
356         this.plan = plan;
357         this.iterations = testIterationCount || plan.iterations || defaultIterationCount;
358         this.isAsync = !!plan.isAsync;
359
360         this.scripts = null;
361
362         this._resourcesPromise = null;
363         this.fetchResources();
364     }
365
366     get name() { return this.plan.name; }
367
368     get runnerCode() {
369         return `
370             let __benchmark = new Benchmark(${this.iterations});
371             let results = [];
372             for (let i = 0; i < ${this.iterations}; i++) {
373                 if (__benchmark.prepareForNextIteration)
374                     __benchmark.prepareForNextIteration();
375
376                 let start = Date.now();
377                 __benchmark.runIteration();
378                 let end = Date.now();
379
380                 results.push(Math.max(1, end - start));
381             }
382             if (__benchmark.validate)
383                 __benchmark.validate();
384             top.currentResolve(results);`;
385     }
386
387     processResults() {
388         throw new Error("Subclasses need to implement this");
389     }
390
391     get score() {
392         throw new Error("Subclasses need to implement this");
393     }
394
395     get prerunCode() { return null; }
396
397     async run() {
398         let code;
399         if (isInBrowser)
400             code = "";
401         else
402             code = [];
403
404         let addScript = (text) => {
405             if (isInBrowser)
406                 code += `<script>${text}</script>`;
407             else
408                 code.push(text);
409         };
410
411         let addScriptWithURL = (url) => {
412             if (isInBrowser)
413                 code += `<script src="${url}"></script>`;
414             else
415                 assert(false, "Should not reach here in CLI");
416         };
417
418         addScript(`const isInBrowser = ${isInBrowser}; let performance = {now: Date.now.bind(Date)};`);
419
420         if (!!this.plan.deterministicRandom) {
421             addScript(`
422                 Math.random = (function() {
423                     var seed = 49734321;
424                     return function() {
425                         // Robert Jenkins' 32 bit integer hash function.
426                         seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
427                         seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
428                         seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
429                         seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
430                         seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
431                         seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
432                         return (seed & 0xfffffff) / 0x10000000;
433                     };
434                 })();
435             `);
436
437         }
438
439         if (this.plan.preload) {
440             let str = "";
441             for (let [variableName, blobUrl] of this.preloads)
442                 str += `const ${variableName} = "${blobUrl}";\n`;
443             addScript(str);
444         }
445
446         let prerunCode = this.prerunCode;
447         if (prerunCode)
448             addScript(prerunCode);
449
450         if (preloadResources) {
451             assert(this.scripts && this.scripts.length === this.plan.files.length);
452
453             for (let text of this.scripts)
454                 addScript(text);
455         } else {
456             for (let file of this.plan.files)
457                 addScriptWithURL(file);
458         }
459
460         let promise = new Promise((resolve, reject) => {
461             currentResolve = resolve;
462             currentReject = reject;
463         });
464
465         if (isInBrowser) {
466             code = `
467                 <script> window.onerror = top.currentReject; </script>
468                 ${code}
469             `;
470         }
471         addScript(this.runnerCode);
472
473         this.startTime = new Date();
474
475         if (RAMification)
476             resetMemoryPeak();
477
478         let magicFrame;
479         try {
480             magicFrame = JetStream.runCode(code);
481         } catch(e) {
482             console.log("Error in runCode: ", e);
483             throw e;
484         }
485         let results = await promise;
486
487         this.endTime = new Date();
488
489         if (RAMification) {
490             let memoryFootprint = MemoryFootprint();
491             this.currentFootprint = memoryFootprint.current;
492             this.peakFootprint = memoryFootprint.peak;
493         }
494
495         this.processResults(results);
496         if (isInBrowser)
497             magicFrame.contentDocument.close();
498     }
499
500     fetchResources() {
501         if (this._resourcesPromise)
502             return this._resourcesPromise;
503
504         let filePromises = preloadResources ? this.plan.files.map((file) => fileLoader.load(file)) : [];
505         let preloads = [];
506         let preloadVariableNames = [];
507
508         if (isInBrowser && this.plan.preload) {
509             for (let prop of Object.getOwnPropertyNames(this.plan.preload)) {
510                 preloadVariableNames.push(prop);
511                 preloads.push(this.plan.preload[prop]);
512             }
513         }
514
515         preloads = preloads.map((file) => fileLoader.load(file));
516
517         let p1 = Promise.all(filePromises).then((texts) => {
518             if (!preloadResources)
519                 return;
520             this.scripts = [];
521             assert(texts.length === this.plan.files.length);
522             for (let text of texts)
523                 this.scripts.push(text);
524         });
525
526         let p2 = Promise.all(preloads).then((data) => {
527             this.preloads = [];
528             this.blobs = [];
529             for (let i = 0; i < data.length; ++i) {
530                 let item = data[i];
531
532                 let blob;
533                 if (typeof item === "string") {
534                     blob = new Blob([item], {type : 'application/javascript'});
535                 } else if (item instanceof ArrayBuffer) {
536                     blob = new Blob([item], {type : 'application/octet-stream'});
537                 } else
538                     throw new Error("Unexpected item!");
539                 
540                 this.blobs.push(blob);
541                 this.preloads.push([preloadVariableNames[i], URL.createObjectURL(blob)]);
542             }
543         });
544
545         this._resourcesPromise = Promise.all([p1, p2]);
546         return this._resourcesPromise;
547     }
548
549     static scoreDescription() { throw new Error("Must be implemented by subclasses."); }
550     scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); }
551
552     updateUIBeforeRun() {
553         if (!isInBrowser) {
554             console.log(`Running ${this.name}:`);
555             return;
556         }
557
558         document.getElementById(`row-for-${this.name}`).classList.add("benchmark-running");
559
560         for (let id of this.scoreIdentifiers())
561             document.getElementById(id).innerHTML = "...";
562     }
563
564     updateUIAfterRun() {
565         if (!isInBrowser)
566             return;
567
568         document.getElementById(`row-for-${this.name}`).classList.remove("benchmark-running");
569     }
570 };
571
572 class DefaultBenchmark extends Benchmark {
573     constructor(...args) {
574         super(...args);
575
576         this.worstCaseCount = this.plan.worstCaseCount || defaultWorstCaseCount;
577         this.firstIteration = null;
578         this.worst4 = null;
579         this.average = null;
580     }
581
582     processResults(results) {
583         function copyArray(a) {
584             let result = [];
585             for (let x of a)
586                 result.push(x);
587             return result;
588         }
589         results = copyArray(results);
590
591         this.firstIteration = results[0];
592
593         results = results.slice(1);
594         results.sort((a, b) => a < b ? 1 : -1);
595         for (let i = 0; i + 1 < results.length; ++i)
596             assert(results[i] >= results[i + 1]);
597
598         let worstCase = [];
599         for (let i = 0; i < this.worstCaseCount; ++i)
600             worstCase.push(results[i]);
601         this.worst4 = mean(worstCase);
602         this.average = mean(results);
603     }
604
605     get score() {
606         return toScore(geomean([this.firstIteration, this.worst4, this.average]));
607     }
608
609     subTimes() {
610         return {
611             "First": this.firstIteration,
612             "Worst": this.worst4,
613             "Average": this.average,
614         };
615     }
616
617     static scoreDescription() {
618         return ["First (ms)", "Worst (ms)", "Average (ms)", "Score"];
619     }
620
621     scoreIdentifiers() {
622         return [firstID(this), worst4ID(this), avgID(this), scoreID(this)];
623     }
624
625     updateUIAfterRun() {
626         super.updateUIAfterRun();
627
628         if (isInBrowser) {
629             document.getElementById(firstID(this)).innerHTML = uiFriendlyNumber(this.firstIteration);
630             document.getElementById(worst4ID(this)).innerHTML = uiFriendlyNumber(this.worst4);
631             document.getElementById(avgID(this)).innerHTML = uiFriendlyNumber(this.average);
632             document.getElementById(scoreID(this)).innerHTML = uiFriendlyNumber(this.score);
633             return;
634         }
635
636         print("    Startup:", uiFriendlyNumber(this.firstIteration), "ms");
637         print("    Worst Case:", uiFriendlyNumber(this.worst4), "ms");
638         print("    Average:", uiFriendlyNumber(this.average), "ms");
639         print("    Score:", uiFriendlyNumber(this.score));
640         if (RAMification) {
641             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
642             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
643         }
644         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
645     }
646 }
647
648 class AsyncBenchmark extends DefaultBenchmark {
649     get runnerCode() {
650         return `
651         async function doRun() {
652             let __benchmark = new Benchmark();
653             let results = [];
654             for (let i = 0; i < ${this.iterations}; i++) {
655                 let start = Date.now();
656                 await __benchmark.runIteration();
657                 let end = Date.now();
658                 results.push(Math.max(1, end - start));
659             }
660             if (__benchmark.validate)
661                 __benchmark.validate();
662             top.currentResolve(results);
663         }
664         doRun();`
665     }
666 };
667
668 class WSLBenchmark extends Benchmark {
669     constructor(...args) {
670         super(...args);
671
672         this.stdlib = null;
673         this.mainRun = null;
674     }
675
676     processResults(results) {
677         this.stdlib = results[0];
678         this.mainRun = results[1];
679     }
680
681     get score() {
682         return toScore(geomean([this.stdlib, this.mainRun]));
683     }
684
685     get runnerCode() {
686         return `
687             let benchmark = new Benchmark();
688             let results = [];
689             {
690                 let start = Date.now();
691                 benchmark.buildStdlib();
692                 results.push(Date.now() - start);
693             }
694
695             {
696                 let start = Date.now();
697                 benchmark.run();
698                 results.push(Date.now() - start);
699             }
700
701             top.currentResolve(results);
702             `;
703     }
704
705     subTimes() {
706         return {
707             "Stdlib": this.stdlib,
708             "MainRun": this.mainRun,
709         };
710     }
711
712     static scoreDescription() {
713         return ["Stdlib (ms)", "MainRun (ms)", "Score"];
714     }
715
716     scoreIdentifiers() {
717         return ["wsl-stdlib-score", "wsl-tests-score", "wsl-score-score"];
718     }
719
720     updateUIAfterRun() {
721         super.updateUIAfterRun();
722
723         if (isInBrowser) {
724             document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyNumber(this.stdlib);
725             document.getElementById("wsl-tests-score").innerHTML = uiFriendlyNumber(this.mainRun);
726             document.getElementById("wsl-score-score").innerHTML = uiFriendlyNumber(this.score);
727             return;
728         }
729
730         print("    Stdlib:", uiFriendlyNumber(this.stdlib), "ms");
731         print("    Tests:", uiFriendlyNumber(this.mainRun), "ms");
732         print("    Score:", uiFriendlyNumber(this.score));
733         if (RAMification) {
734             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
735             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
736         }
737         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
738     }
739 };
740
741 class WasmBenchmark extends Benchmark {
742     constructor(...args) {
743         super(...args);
744
745         this.startupTime = null;
746         this.runTime = null;
747     }
748
749     processResults(results) {
750         this.startupTime = results[0];
751         this.runTime = results[1];
752     }
753
754     get score() {
755         return toScore(geomean([this.startupTime, this.runTime]));
756     }
757
758     get wasmPath() {
759         return this.plan.wasmPath;
760     }
761
762     get prerunCode() {
763         let str = `
764             let verbose = false;
765
766             let compileTime = null;
767             let runTime = null;
768
769             let globalObject = this;
770
771             globalObject.benchmarkTime = Date.now.bind(Date);
772
773             globalObject.reportCompileTime = (t) => {
774                 if (compileTime !== null)
775                     throw new Error("called report compile time twice");
776                 compileTime = t;
777             };
778
779             globalObject.reportRunTime = (t) => {
780                 if (runTime !== null)
781                     throw new Error("called report run time twice")
782                 runTime = t;
783                 top.currentResolve([compileTime, runTime]);
784             };
785
786             abort = quit = function() {
787                 if (verbose)
788                     console.log('Intercepted quit/abort');
789             };
790
791             oldPrint = globalObject.print;
792             globalObject.print = globalObject.printErr = (...args) => {
793                 if (verbose)
794                     console.log('Intercepted print: ', ...args);
795             };
796
797             let Module = {
798                 preRun: [],
799                 postRun: [],
800                 print: function() { },
801                 printErr: function() { },
802                 setStatus: function(text) {
803                 },
804                 totalDependencies: 0,
805                 monitorRunDependencies: function(left) {
806                     this.totalDependencies = Math.max(this.totalDependencies, left);
807                     Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
808                 }
809             };
810             globalObject.Module = Module;
811             `;
812         return str;
813     }
814
815     get runnerCode() { 
816         let str = "";
817         if (isInBrowser) {
818             str += `
819                 var xhr = new XMLHttpRequest();
820                 xhr.open('GET', wasmBlobURL, true);
821                 xhr.responseType = 'arraybuffer';
822                 xhr.onload = function() {
823                     Module.wasmBinary = xhr.response;
824                     doRun();
825                 };
826                 xhr.send(null);
827             `;
828         } else {
829             str += `
830             Module.wasmBinary = read("${this.wasmPath}", "binary");
831             globalObject.read = (...args) => {
832                 console.log("should not be inside read: ", ...args);
833                 throw new Error;
834             };
835
836             Module.setStatus = null;
837             Module.monitorRunDependencies = null;
838
839             Promise.resolve(42).then(() => {
840                 try {
841                     doRun();
842                 } catch(e) {
843                     console.log("error running wasm:", e);
844                     throw e;
845                 }
846             })
847             `;
848         }
849         return str;
850     }
851
852     subTimes() {
853         return {
854             "Startup": this.startupTime,
855             "Runtime": this.runTime,
856         };
857     }
858
859     static scoreDescription() {
860         return ["Startup (ms)", "Runtime (ms)", "Score"];
861     }
862
863     get startupID() {
864         return `wasm-startup-id${this.name}`;
865     }
866     get runID() {
867         return `wasm-run-id${this.name}`;
868     }
869     get scoreID() {
870         return `wasm-score-id${this.name}`;
871     }
872
873     scoreIdentifiers() {
874         return [this.startupID, this.runID, this.scoreID];
875     }
876
877     updateUIAfterRun() {
878         super.updateUIAfterRun();
879
880         if (isInBrowser) {
881             document.getElementById(this.startupID).innerHTML = uiFriendlyNumber(this.startupTime);
882             document.getElementById(this.runID).innerHTML = uiFriendlyNumber(this.runTime);
883             document.getElementById(this.scoreID).innerHTML = uiFriendlyNumber(this.score);
884             return;
885         }
886         print("    Startup:", uiFriendlyNumber(this.startupTime), "ms");
887         print("    Run time:", uiFriendlyNumber(this.runTime), "ms");
888         if (RAMification) {
889             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
890             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
891         }
892         print("    Score:", uiFriendlyNumber(this.score));
893     }
894 };
895
896 const ARESGroup = Symbol.for("ARES");
897 const CDJSGroup = Symbol.for("CDJS");
898 const CodeLoadGroup = Symbol.for("CodeLoad");
899 const LuaJSFightGroup = Symbol.for("LuaJSFight");
900 const OctaneGroup = Symbol.for("Octane");
901 const RexBenchGroup = Symbol.for("RexBench");
902 const SeaMonsterGroup = Symbol.for("SeaMonster");
903 const SimpleGroup = Symbol.for("Simple");
904 const SunSpiderGroup = Symbol.for("SunSpider");
905 const WasmGroup = Symbol.for("Wasm");
906 const WorkerTestsGroup = Symbol.for("WorkerTests");
907 const WSLGroup = Symbol.for("WSL");
908 const WTBGroup = Symbol.for("WTB");
909
910
911 let testPlans = [
912     // ARES
913     {
914         name: "Air",
915         files: [
916             "./ARES-6/Air/symbols.js"
917             , "./ARES-6/Air/tmp_base.js"
918             , "./ARES-6/Air/arg.js"
919             , "./ARES-6/Air/basic_block.js"
920             , "./ARES-6/Air/code.js"
921             , "./ARES-6/Air/frequented_block.js"
922             , "./ARES-6/Air/inst.js"
923             , "./ARES-6/Air/opcode.js"
924             , "./ARES-6/Air/reg.js"
925             , "./ARES-6/Air/stack_slot.js"
926             , "./ARES-6/Air/tmp.js"
927             , "./ARES-6/Air/util.js"
928             , "./ARES-6/Air/custom.js"
929             , "./ARES-6/Air/liveness.js"
930             , "./ARES-6/Air/insertion_set.js"
931             , "./ARES-6/Air/allocate_stack.js"
932             , "./ARES-6/Air/payload-gbemu-executeIteration.js"
933             , "./ARES-6/Air/payload-imaging-gaussian-blur-gaussianBlur.js"
934             , "./ARES-6/Air/payload-airjs-ACLj8C.js"
935             , "./ARES-6/Air/payload-typescript-scanIdentifier.js"
936             , "./ARES-6/Air/benchmark.js"
937         ],
938         testGroup: ARESGroup
939     },
940     {
941         name: "Basic",
942         files: [
943             "./ARES-6/Basic/ast.js"
944             , "./ARES-6/Basic/basic.js"
945             , "./ARES-6/Basic/caseless_map.js"
946             , "./ARES-6/Basic/lexer.js"
947             , "./ARES-6/Basic/number.js"
948             , "./ARES-6/Basic/parser.js"
949             , "./ARES-6/Basic/random.js"
950             , "./ARES-6/Basic/state.js"
951             , "./ARES-6/Basic/util.js"
952             , "./ARES-6/Basic/benchmark.js"
953         ],
954         testGroup: ARESGroup
955     },
956     {
957         name: "ML",
958         files: [
959             "./ARES-6/ml/index.js"
960             , "./ARES-6/ml/benchmark.js"
961         ],
962         iterations: 60,
963         testGroup: ARESGroup
964     },
965     {
966         name: "Babylon",
967         files: [
968             "./ARES-6/Babylon/index.js"
969             , "./ARES-6/Babylon/benchmark.js"
970         ],
971         preload: {
972             airBlob: "./ARES-6/Babylon/air-blob.js",
973             basicBlob: "./ARES-6/Babylon/basic-blob.js",
974             inspectorBlob: "./ARES-6/Babylon/inspector-blob.js",
975             babylonBlob: "./ARES-6/Babylon/babylon-blob.js"
976         },
977         testGroup: ARESGroup
978     },
979     // CDJS
980     {
981         name: "cdjs",
982         files: [
983             "./cdjs/constants.js"
984             , "./cdjs/util.js"
985             , "./cdjs/red_black_tree.js"
986             , "./cdjs/call_sign.js"
987             , "./cdjs/vector_2d.js"
988             , "./cdjs/vector_3d.js"
989             , "./cdjs/motion.js"
990             , "./cdjs/reduce_collision_set.js"
991             , "./cdjs/simulator.js"
992             , "./cdjs/collision.js"
993             , "./cdjs/collision_detector.js"
994             , "./cdjs/benchmark.js"
995         ],
996         iterations: 60,
997         worstCaseCount: 3,
998         testGroup: CDJSGroup
999     },
1000     // CodeLoad
1001     {
1002         name: "first-inspector-code-load",
1003         files: [
1004             "./code-load/code-first-load.js"
1005         ],
1006         preload: {
1007             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1008         },
1009         testGroup: CodeLoadGroup
1010     },
1011     {
1012         name: "multi-inspector-code-load",
1013         files: [
1014             "./code-load/code-multi-load.js"
1015         ],
1016         preload: {
1017             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1018         },
1019         testGroup: CodeLoadGroup
1020     },
1021     // Octane
1022     {
1023         name: "Box2D",
1024         files: [
1025             "./Octane/box2d.js"
1026         ],
1027         deterministicRandom: true,
1028         testGroup: OctaneGroup
1029     },
1030     {
1031         name: "octane-code-load",
1032         files: [
1033             "./Octane/code-first-load.js"
1034         ],
1035         deterministicRandom: true,
1036         testGroup: OctaneGroup
1037     },
1038     {
1039         name: "crypto",
1040         files: [
1041             "./Octane/crypto.js"
1042         ],
1043         deterministicRandom: true,
1044         testGroup: OctaneGroup
1045     },
1046     {
1047         name: "delta-blue",
1048         files: [
1049             "./Octane/deltablue.js"
1050         ],
1051         deterministicRandom: true,
1052         testGroup: OctaneGroup
1053     },
1054     {
1055         name: "earley-boyer",
1056         files: [
1057             "./Octane/earley-boyer.js"
1058         ],
1059         deterministicRandom: true,
1060         testGroup: OctaneGroup
1061     },
1062     {
1063         name: "gbemu",
1064         files: [
1065             "./Octane/gbemu-part1.js"
1066             , "./Octane/gbemu-part2.js"
1067         ],
1068         deterministicRandom: true,
1069         testGroup: OctaneGroup
1070     },
1071     {
1072         name: "mandreel",
1073         files: [
1074             "./Octane/mandreel.js"
1075         ],
1076         iterations: 80,
1077         deterministicRandom: true,
1078         testGroup: OctaneGroup
1079     },
1080     {
1081         name: "navier-stokes",
1082         files: [
1083             "./Octane/navier-stokes.js"
1084         ],
1085         deterministicRandom: true,
1086         testGroup: OctaneGroup
1087     },
1088     {
1089         name: "pdfjs",
1090         files: [
1091             "./Octane/pdfjs.js"
1092         ],
1093         deterministicRandom: true,
1094         testGroup: OctaneGroup
1095     },
1096     {
1097         name: "raytrace",
1098         files: [
1099             "./Octane/raytrace.js"
1100         ],
1101         deterministicRandom: true,
1102         testGroup: OctaneGroup
1103     },
1104     {
1105         name: "regexp",
1106         files: [
1107             "./Octane/regexp.js"
1108         ],
1109         deterministicRandom: true,
1110         testGroup: OctaneGroup
1111     },
1112     {
1113         name: "richards",
1114         files: [
1115             "./Octane/richards.js"
1116         ],
1117         deterministicRandom: true,
1118         testGroup: OctaneGroup
1119     },
1120     {
1121         name: "splay",
1122         files: [
1123             "./Octane/splay.js"
1124         ],
1125         deterministicRandom: true,
1126         testGroup: OctaneGroup
1127     },
1128     {
1129         name: "typescript",
1130         files: [
1131             "./Octane/typescript-compiler.js"
1132             , "./Octane/typescript-input.js"
1133             , "./Octane/typescript.js"
1134         ],
1135         iterations: 15,
1136         worstCaseCount: 2,
1137         deterministicRandom: true,
1138         testGroup: OctaneGroup
1139     },
1140     {
1141         name: "octane-zlib",
1142         files: [
1143             "./Octane/zlib-data.js"
1144             , "./Octane/zlib.js"
1145         ],
1146         iterations: 15,
1147         worstCaseCount: 2,
1148         deterministicRandom: true,
1149         testGroup: OctaneGroup
1150     },
1151     // RexBench
1152     {
1153         name: "FlightPlanner",
1154         files: [
1155             "./RexBench/FlightPlanner/airways.js"
1156             , "./RexBench/FlightPlanner/waypoints.js"
1157             , "./RexBench/FlightPlanner/flight_planner.js"
1158             , "./RexBench/FlightPlanner/expectations.js"
1159             , "./RexBench/FlightPlanner/benchmark.js"
1160         ],
1161         testGroup: RexBenchGroup
1162     },
1163     {
1164         name: "OfflineAssembler",
1165         files: [
1166             "./RexBench/OfflineAssembler/registers.js"
1167             , "./RexBench/OfflineAssembler/instructions.js"
1168             , "./RexBench/OfflineAssembler/ast.js"
1169             , "./RexBench/OfflineAssembler/parser.js"
1170             , "./RexBench/OfflineAssembler/file.js"
1171             , "./RexBench/OfflineAssembler/LowLevelInterpreter.js"
1172             , "./RexBench/OfflineAssembler/LowLevelInterpreter32_64.js"
1173             , "./RexBench/OfflineAssembler/LowLevelInterpreter64.js"
1174             , "./RexBench/OfflineAssembler/InitBytecodes.js"
1175             , "./RexBench/OfflineAssembler/expected.js"
1176             , "./RexBench/OfflineAssembler/benchmark.js"
1177         ],
1178         iterations: 80,
1179         testGroup: RexBenchGroup
1180     },
1181     {
1182         name: "UniPoker",
1183         files: [
1184             "./RexBench/UniPoker/poker.js"
1185             , "./RexBench/UniPoker/expected.js"
1186             , "./RexBench/UniPoker/benchmark.js"
1187         ],
1188         deterministicRandom: true,
1189         testGroup: RexBenchGroup
1190     },
1191     // Simple
1192     {
1193         name: "async-fs",
1194         files: [
1195             "./simple/file-system.js"
1196         ],
1197         iterations: 40,
1198         worstCaseCount: 3,
1199         benchmarkClass: AsyncBenchmark,
1200         testGroup: SimpleGroup
1201     },
1202     {
1203         name: "float-mm.c",
1204         files: [
1205             "./simple/float-mm.c.js"
1206         ],
1207         iterations: 15,
1208         worstCaseCount: 2,
1209         testGroup: SimpleGroup
1210     },
1211     {
1212         name: "hash-map",
1213         files: [
1214             "./simple/hash-map.js"
1215         ],
1216         testGroup: SimpleGroup
1217     },
1218     // SeaMonster
1219     {
1220         name: "ai-astar",
1221         files: [
1222             "./SeaMonster/ai-astar.js"
1223         ],
1224         testGroup: SeaMonsterGroup
1225     },
1226     {
1227         name: "gaussian-blur",
1228         files: [
1229             "./SeaMonster/gaussian-blur.js"
1230         ],
1231         testGroup: SeaMonsterGroup
1232     },
1233     {
1234         name: "stanford-crypto-aes",
1235         files: [
1236             "./SeaMonster/sjlc.js"
1237             , "./SeaMonster/stanford-crypto-aes.js"
1238         ],
1239         testGroup: SeaMonsterGroup
1240     },
1241     {
1242         name: "stanford-crypto-pbkdf2",
1243         files: [
1244             "./SeaMonster/sjlc.js"
1245             , "./SeaMonster/stanford-crypto-pbkdf2.js"
1246         ],
1247         testGroup: SeaMonsterGroup
1248     },
1249     {
1250         name: "stanford-crypto-sha256",
1251         files: [
1252             "./SeaMonster/sjlc.js"
1253             , "./SeaMonster/stanford-crypto-sha256.js"
1254         ],
1255         testGroup: SeaMonsterGroup
1256     },
1257     {
1258         name: "json-stringify-inspector",
1259         files: [
1260             "./SeaMonster/inspector-json-payload.js"
1261             , "./SeaMonster/json-stringify-inspector.js"
1262         ],
1263         iterations: 20,
1264         worstCaseCount: 2,
1265         testGroup: SeaMonsterGroup
1266     },
1267     {
1268         name: "json-parse-inspector",
1269         files: [
1270             "./SeaMonster/inspector-json-payload.js"
1271             , "./SeaMonster/json-parse-inspector.js"
1272         ],
1273         iterations: 20,
1274         worstCaseCount: 2,
1275         testGroup: SeaMonsterGroup
1276     },
1277     // Wasm
1278     {
1279         name: "HashSet-wasm",
1280         wasmPath: "./wasm/HashSet.wasm",
1281         files: [
1282             "./wasm/HashSet.js"
1283         ],
1284         preload: {
1285             wasmBlobURL: "./wasm/HashSet.wasm"
1286         },
1287         benchmarkClass: WasmBenchmark,
1288         testGroup: WasmGroup
1289     },
1290     {
1291         name: "tsf-wasm",
1292         wasmPath: "./wasm/tsf.wasm",
1293         files: [
1294             "./wasm/tsf.js"
1295         ],
1296         preload: {
1297             wasmBlobURL: "./wasm/tsf.wasm" 
1298         },
1299         benchmarkClass: WasmBenchmark,
1300         testGroup: WasmGroup
1301     },
1302     {
1303         name: "quicksort-wasm",
1304         wasmPath: "./wasm/quicksort.wasm",
1305         files: [
1306             "./wasm/quicksort.js"
1307         ],
1308         preload: {
1309             wasmBlobURL: "./wasm/quicksort.wasm"
1310         },
1311         benchmarkClass: WasmBenchmark,
1312         testGroup: WasmGroup
1313     },
1314     {
1315         name: "gcc-loops-wasm",
1316         wasmPath: "./wasm/gcc-loops.wasm",
1317         files: [
1318             "./wasm/gcc-loops.js"
1319         ],
1320         preload: {
1321             wasmBlobURL: "./wasm/gcc-loops.wasm"
1322         },
1323         benchmarkClass: WasmBenchmark,
1324         testGroup: WasmGroup
1325     },
1326     {
1327         name: "richards-wasm",
1328         wasmPath: "./wasm/richards.wasm",
1329         files: [
1330             "./wasm/richards.js"
1331         ],
1332         preload: {
1333             wasmBlobURL: "./wasm/richards.wasm"
1334         },
1335         benchmarkClass: WasmBenchmark,
1336         testGroup: WasmGroup
1337     },
1338     // WorkerTests
1339     {
1340         name: "bomb-workers",
1341         files: [
1342             "./worker/bomb.js"
1343         ],
1344         iterations: 80,
1345         preload: {
1346             rayTrace3D: "./worker/bomb-subtests/3d-raytrace.js"
1347             , accessNbody: "./worker/bomb-subtests/access-nbody.js"
1348             , morph3D: "./worker/bomb-subtests/3d-morph.js"
1349             , cube3D: "./worker/bomb-subtests/3d-cube.js"
1350             , accessFunnkuch: "./worker/bomb-subtests/access-fannkuch.js"
1351             , accessBinaryTrees: "./worker/bomb-subtests/access-binary-trees.js"
1352             , accessNsieve: "./worker/bomb-subtests/access-nsieve.js"
1353             , bitopsBitwiseAnd: "./worker/bomb-subtests/bitops-bitwise-and.js"
1354             , bitopsNsieveBits: "./worker/bomb-subtests/bitops-nsieve-bits.js"
1355             , controlflowRecursive: "./worker/bomb-subtests/controlflow-recursive.js"
1356             , bitops3BitBitsInByte: "./worker/bomb-subtests/bitops-3bit-bits-in-byte.js"
1357             , botopsBitsInByte: "./worker/bomb-subtests/bitops-bits-in-byte.js"
1358             , cryptoAES: "./worker/bomb-subtests/crypto-aes.js"
1359             , cryptoMD5: "./worker/bomb-subtests/crypto-md5.js"
1360             , cryptoSHA1: "./worker/bomb-subtests/crypto-sha1.js"
1361             , dateFormatTofte: "./worker/bomb-subtests/date-format-tofte.js"
1362             , dateFormatXparb: "./worker/bomb-subtests/date-format-xparb.js"
1363             , mathCordic: "./worker/bomb-subtests/math-cordic.js"
1364             , mathPartialSums: "./worker/bomb-subtests/math-partial-sums.js"
1365             , mathSpectralNorm: "./worker/bomb-subtests/math-spectral-norm.js"
1366             , stringBase64: "./worker/bomb-subtests/string-base64.js"
1367             , stringFasta: "./worker/bomb-subtests/string-fasta.js"
1368             , stringValidateInput: "./worker/bomb-subtests/string-validate-input.js"
1369             , stringTagcloud: "./worker/bomb-subtests/string-tagcloud.js"
1370             , stringUnpackCode: "./worker/bomb-subtests/string-unpack-code.js"
1371             , regexpDNA: "./worker/bomb-subtests/regexp-dna.js"
1372         },
1373         benchmarkClass: AsyncBenchmark,
1374         testGroup: WorkerTestsGroup
1375     },
1376     {
1377         name: "segmentation",
1378         files: [
1379             "./worker/segmentation.js"
1380         ],
1381         preload: {
1382             asyncTaskBlob: "./worker/async-task.js"
1383         },
1384         iterations: 36,
1385         worstCaseCount: 3,
1386         benchmarkClass: AsyncBenchmark,
1387         testGroup: WorkerTestsGroup
1388     },
1389     // WSL
1390     {
1391         name: "WSL",
1392         files: ["./WSL/Node.js" ,"./WSL/Type.js" ,"./WSL/ReferenceType.js" ,"./WSL/Value.js" ,"./WSL/Expression.js" ,"./WSL/Rewriter.js" ,"./WSL/Visitor.js" ,"./WSL/CreateLiteral.js" ,"./WSL/CreateLiteralType.js" ,"./WSL/PropertyAccessExpression.js" ,"./WSL/AddressSpace.js" ,"./WSL/AnonymousVariable.js" ,"./WSL/ArrayRefType.js" ,"./WSL/ArrayType.js" ,"./WSL/Assignment.js" ,"./WSL/AutoWrapper.js" ,"./WSL/Block.js" ,"./WSL/BoolLiteral.js" ,"./WSL/Break.js" ,"./WSL/CallExpression.js" ,"./WSL/CallFunction.js" ,"./WSL/Check.js" ,"./WSL/CheckLiteralTypes.js" ,"./WSL/CheckLoops.js" ,"./WSL/CheckRecursiveTypes.js" ,"./WSL/CheckRecursion.js" ,"./WSL/CheckReturns.js" ,"./WSL/CheckUnreachableCode.js" ,"./WSL/CheckWrapped.js" ,"./WSL/Checker.js" ,"./WSL/CloneProgram.js" ,"./WSL/CommaExpression.js" ,"./WSL/ConstexprFolder.js" ,"./WSL/ConstexprTypeParameter.js" ,"./WSL/Continue.js" ,"./WSL/ConvertPtrToArrayRefExpression.js" ,"./WSL/DereferenceExpression.js" ,"./WSL/DoWhileLoop.js" ,"./WSL/DotExpression.js" ,"./WSL/DoubleLiteral.js" ,"./WSL/DoubleLiteralType.js" ,"./WSL/EArrayRef.js" ,"./WSL/EBuffer.js" ,"./WSL/EBufferBuilder.js" ,"./WSL/EPtr.js" ,"./WSL/EnumLiteral.js" ,"./WSL/EnumMember.js" ,"./WSL/EnumType.js" ,"./WSL/EvaluationCommon.js" ,"./WSL/Evaluator.js" ,"./WSL/ExpressionFinder.js" ,"./WSL/ExternalOrigin.js" ,"./WSL/Field.js" ,"./WSL/FindHighZombies.js" ,"./WSL/FlattenProtocolExtends.js" ,"./WSL/FlattenedStructOffsetGatherer.js" ,"./WSL/FloatLiteral.js" ,"./WSL/FloatLiteralType.js" ,"./WSL/FoldConstexprs.js" ,"./WSL/ForLoop.js" ,"./WSL/Func.js" ,"./WSL/FuncDef.js" ,"./WSL/FuncInstantiator.js" ,"./WSL/FuncParameter.js" ,"./WSL/FunctionLikeBlock.js" ,"./WSL/HighZombieFinder.js" ,"./WSL/IdentityExpression.js" ,"./WSL/IfStatement.js" ,"./WSL/IndexExpression.js" ,"./WSL/InferTypesForCall.js" ,"./WSL/Inline.js" ,"./WSL/Inliner.js" ,"./WSL/InstantiateImmediates.js" ,"./WSL/IntLiteral.js" ,"./WSL/IntLiteralType.js" ,"./WSL/Intrinsics.js" ,"./WSL/LateChecker.js" ,"./WSL/Lexer.js" ,"./WSL/LexerToken.js" ,"./WSL/LiteralTypeChecker.js" ,"./WSL/LogicalExpression.js" ,"./WSL/LogicalNot.js" ,"./WSL/LoopChecker.js" ,"./WSL/MakeArrayRefExpression.js" ,"./WSL/MakePtrExpression.js" ,"./WSL/NameContext.js" ,"./WSL/NameFinder.js" ,"./WSL/NameResolver.js" ,"./WSL/NativeFunc.js" ,"./WSL/NativeFuncInstance.js" ,"./WSL/NativeType.js" ,"./WSL/NativeTypeInstance.js" ,"./WSL/NormalUsePropertyResolver.js" ,"./WSL/NullLiteral.js" ,"./WSL/NullType.js" ,"./WSL/OriginKind.js" ,"./WSL/OverloadResolutionFailure.js" ,"./WSL/Parse.js" ,"./WSL/Prepare.js" ,"./WSL/Program.js" ,"./WSL/ProgramWithUnnecessaryThingsRemoved.js" ,"./WSL/PropertyResolver.js" ,"./WSL/Protocol.js" ,"./WSL/ProtocolDecl.js" ,"./WSL/ProtocolFuncDecl.js" ,"./WSL/ProtocolRef.js" ,"./WSL/PtrType.js" ,"./WSL/ReadModifyWriteExpression.js" ,"./WSL/RecursionChecker.js" ,"./WSL/RecursiveTypeChecker.js" ,"./WSL/ResolveNames.js" ,"./WSL/ResolveOverloadImpl.js" ,"./WSL/ResolveProperties.js" ,"./WSL/ResolveTypeDefs.js" ,"./WSL/Return.js" ,"./WSL/ReturnChecker.js" ,"./WSL/ReturnException.js" ,"./WSL/StandardLibrary.js" ,"./WSL/StatementCloner.js" ,"./WSL/StructLayoutBuilder.js" ,"./WSL/StructType.js" ,"./WSL/Substitution.js" ,"./WSL/SwitchCase.js" ,"./WSL/SwitchStatement.js" ,"./WSL/SynthesizeEnumFunctions.js" ,"./WSL/SynthesizeStructAccessors.js" ,"./WSL/TrapStatement.js" ,"./WSL/TypeDef.js" ,"./WSL/TypeDefResolver.js" ,"./WSL/TypeOrVariableRef.js" ,"./WSL/TypeParameterRewriter.js" ,"./WSL/TypeRef.js" ,"./WSL/TypeVariable.js" ,"./WSL/TypeVariableTracker.js" ,"./WSL/TypedValue.js" ,"./WSL/UintLiteral.js" ,"./WSL/UintLiteralType.js" ,"./WSL/UnificationContext.js" ,"./WSL/UnreachableCodeChecker.js" ,"./WSL/VariableDecl.js" ,"./WSL/VariableRef.js" ,"./WSL/VisitingSet.js" ,"./WSL/WSyntaxError.js" ,"./WSL/WTrapError.js" ,"./WSL/WTypeError.js" ,"./WSL/WhileLoop.js" ,"./WSL/WrapChecker.js", "./WSL/Test.js"],
1393         benchmarkClass: WSLBenchmark,
1394         testGroup: WSLGroup
1395     }
1396 ];
1397
1398 // LuaJSFight tests
1399 let luaJSFightTests = [
1400     "hello_world"
1401     , "list_search"
1402     , "lists"
1403     , "string_lists"
1404 ];
1405 for (let test of luaJSFightTests) {
1406     testPlans.push({
1407         name: `${test}-LJF`,
1408         files: [
1409             `./LuaJSFight/${test}.js`
1410         ],
1411         testGroup: LuaJSFightGroup
1412     });
1413 }
1414
1415 // SunSpider tests
1416 let sunSpiderTests = [
1417     "3d-cube"
1418     , "3d-raytrace"
1419     , "base64"
1420     , "crypto-aes"
1421     , "crypto-md5"
1422     , "crypto-sha1"
1423     , "date-format-tofte"
1424     , "date-format-xparb"
1425     , "n-body"
1426     , "regex-dna"
1427     , "string-unpack-code"
1428     , "tagcloud"
1429 ];
1430 for (let test of sunSpiderTests) {
1431     testPlans.push({
1432         name: `${test}-SP`,
1433         files: [
1434             `./SunSpider/${test}.js`
1435         ],
1436         testGroup: SunSpiderGroup
1437     });
1438 }
1439
1440 // WTB (Web Tooling Benchmark) tests
1441 let WTBTests = [
1442     "acorn"
1443     , "babylon"
1444     , "chai"
1445     , "coffeescript"
1446     , "espree"
1447     , "jshint"
1448     , "lebab"
1449     , "prepack"
1450     , "uglify-js"
1451 ];
1452 for (let name of WTBTests) {
1453     testPlans.push({
1454         name: `${name}-wtb`,
1455         files: [
1456             isInBrowser ? "./web-tooling-benchmark/browser.js" : "./web-tooling-benchmark/cli.js"
1457             , `./web-tooling-benchmark/${name}.js`
1458         ],
1459         iterations: 5,
1460         worstCaseCount: 1,
1461         testGroup: WTBGroup
1462     });
1463 }
1464
1465
1466 let testsByName = new Map();
1467 let testsByGroup = new Map();
1468
1469 for (let plan of testPlans) {
1470     let testName = plan.name;
1471
1472     if (testsByName.has(plan.name))
1473         throw "Duplicate test plan with name \"" + testName + "\"";
1474     else
1475         testsByName.set(testName, plan);
1476
1477     let group = plan.testGroup;
1478
1479     if (testsByGroup.has(group))
1480         testsByGroup.get(group).push(testName);
1481     else
1482         testsByGroup.set(group, [testName]);
1483 }
1484
1485 this.JetStream = new Driver();
1486
1487 function addTestByName(testName)
1488 {
1489     let plan = testsByName.get(testName);
1490
1491     if (plan)
1492         JetStream.addPlan(plan, plan.benchmarkClass);
1493     else
1494         throw "Couldn't find test named \"" +  testName + "\"";
1495 }
1496
1497 function addTestsByGroup(group)
1498 {
1499     let testList = testsByGroup.get(group);
1500
1501     if (!testList)
1502         throw "Couldn't find test group named: \"" + Symbol.keyFor(group) + "\"";
1503
1504     for (let testName of testList)
1505         addTestByName(testName);
1506 }
1507
1508 function processTestList(testList)
1509 {
1510     let tests = [];
1511
1512     if (testList instanceof Array)
1513         tests = testList;
1514     else
1515         tests = testList.split(/[\s,]/);
1516
1517     for (let testName of tests) {
1518         let groupTest = testsByGroup.get(Symbol.for(testName));
1519
1520         if (groupTest) {
1521             for (let testName of groupTest)
1522                 addTestByName(testName);
1523         } else
1524             addTestByName(testName);
1525     }
1526 }
1527
1528 let runOctane = true;
1529 let runARES = true;
1530 let runWSL = true;
1531 let runRexBench = true;
1532 let runWTB = true;
1533 let runSunSpider = true;
1534 let runSimple = true;
1535 let runCDJS = true;
1536 let runWorkerTests = !!isInBrowser;
1537 let runSeaMonster = true;
1538 let runCodeLoad = true;
1539 let runWasm = true;
1540
1541 if (false) {
1542     runOctane = false;
1543     runARES = false;
1544     runWSL = false;
1545     runRexBench = false;
1546     runWTB = false;
1547     runSunSpider = false;
1548     runSimple = false;
1549     runCDJS = false;
1550     runWorkerTests = false;
1551     runSeaMonster = false;
1552     runCodeLoad = false;
1553     runWasm = false;
1554 }
1555
1556 if (typeof testList !== "undefined") {
1557     processTestList(testList);
1558 } else {
1559     if (runARES)
1560         addTestsByGroup(ARESGroup);
1561
1562     if (runCDJS)
1563         addTestsByGroup(CDJSGroup);
1564
1565     if (runCodeLoad)
1566         addTestsByGroup(CodeLoadGroup);
1567
1568     if (runOctane)
1569         addTestsByGroup(OctaneGroup);
1570
1571     if (runRexBench)
1572         addTestsByGroup(RexBenchGroup);
1573
1574     if (runSeaMonster)
1575         addTestsByGroup(SeaMonsterGroup);
1576
1577     if (runSimple)
1578         addTestsByGroup(SimpleGroup);
1579
1580     if (runSunSpider)
1581         addTestsByGroup(SunSpiderGroup);
1582
1583     if (runWasm)
1584         addTestsByGroup(WasmGroup);
1585
1586     if (runWorkerTests)
1587         addTestsByGroup(WorkerTestsGroup);
1588
1589     if (runWSL)
1590         addTestsByGroup(WSLGroup);
1591
1592     if (runWTB)
1593         addTestsByGroup(WTBGroup);
1594 }