Enable HAVE_AVFOUNDATION_VIDEO_OUTPUT on PLATFORM(IOSMAC)
[WebKit-https.git] / PerformanceTests / JetStream2 / JetStreamDriver.js
1 "use strict";
2
3 /*
4  * Copyright (C) 2018 Apple Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
19  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
25  * THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 const preloadResources = !isInBrowser;
29 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
30
31 if (typeof RAMification === "undefined")
32     var RAMification = false;
33
34 if (typeof testIterationCount === "undefined")
35     var testIterationCount = undefined;
36
37 // Used for the promise representing the current benchmark run.
38 this.currentResolve = null;
39 this.currentReject = null;
40
41 const defaultIterationCount = 120;
42 const defaultWorstCaseCount = 4;
43
44 function assert(b, m = "") {
45     if (!b)
46         throw new Error("Bad assertion: " + m);
47 }
48
49 function firstID(benchmark) {
50     return `results-cell-${benchmark.name}-first`;
51 }
52
53 function worst4ID(benchmark) {
54     return `results-cell-${benchmark.name}-worst4`;
55 }
56
57 function avgID(benchmark) {
58     return `results-cell-${benchmark.name}-avg`;
59 }
60
61 function scoreID(benchmark) {
62     return `results-cell-${benchmark.name}-score`;
63 }
64
65 function mean(values) {
66     assert(values instanceof Array);
67     let sum = 0;
68     for (let x of values)
69         sum += x;
70     return sum / values.length;
71 }
72
73 function geomean(values) {
74     assert(values instanceof Array);
75     let product = 1;
76     for (let x of values)
77         product *= x;
78     return product ** (1 / values.length);
79 }
80
81 function toScore(timeValue) {
82     return 5000 / timeValue;
83 }
84
85 function toTimeValue(score) {
86     return 5000 / score;
87 }
88
89 function updateUI() {
90     return new Promise((resolve) => {
91         if (isInBrowser)
92             requestAnimationFrame(() => setTimeout(resolve, 0));
93         else
94             resolve();
95     });
96 }
97
98 function uiFriendlyNumber(num) {
99     if (Number.isInteger(num))
100         return num;
101     return num.toFixed(3);
102 }
103
104 function uiFriendlyDuration(time)
105 {
106     let minutes = time.getMinutes();
107     let seconds = time.getSeconds();
108     let milliSeconds = time.getMilliseconds();
109     let result = "" + minutes + ":";
110
111     result = result + (seconds < 10 ? "0" : "") + seconds + ".";
112     result = result + (milliSeconds < 10 ? "00" : (milliSeconds < 100 ? "0" : "")) + milliSeconds;
113
114     return result;
115 }
116
117 const fileLoader = (function() {
118     class Loader {
119         constructor() {
120             this.requests = new Map;
121         }
122
123         async _loadInternal(url) {
124             if (!isInBrowser)
125                 return Promise.resolve(readFile(url));
126
127             let fetchResponse = await fetch(new Request(url));
128             if (url.indexOf(".js") !== -1)
129                 return await fetchResponse.text();
130             else if (url.indexOf(".wasm") !== -1)
131                 return await fetchResponse.arrayBuffer();
132
133             throw new Error("should not be reached!");
134         }
135
136         async load(url) {
137             if (this.requests.has(url))
138                 return (await this.requests.get(url));
139
140             let promise = this._loadInternal(url);
141             this.requests.set(url, promise);
142             return (await promise);
143         }
144     }
145     return new Loader;
146 })();
147
148 class Driver {
149     constructor() {
150         this.benchmarks = [];
151     }
152
153     addPlan(plan, BenchmarkClass = DefaultBenchmark) {
154         this.benchmarks.push(new BenchmarkClass(plan));
155     }
156
157     async start() {
158         let statusElement = false;
159         let summaryElement = false;
160         if (isInBrowser) {
161             statusElement = document.getElementById("status");
162             summaryElement = document.getElementById("result-summary");
163             statusElement.innerHTML = `<label>Running...</label>`;
164         } else {
165             console.log("Starting JetStream2");
166         }
167
168         await updateUI();
169
170         let start = Date.now();
171         for (let benchmark of this.benchmarks) {
172             benchmark.updateUIBeforeRun();
173
174             await updateUI();
175
176             try {
177
178                 await benchmark.run();
179             } catch(e) {
180                 JetStream.reportError(benchmark);
181                 throw e;
182             }
183
184             benchmark.updateUIAfterRun();
185         }
186
187         let totalTime = Date.now() - start;
188         if (measureTotalTimeAsSubtest) {
189             if (isInBrowser)
190                 document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime);
191             else
192                 console.log("Total time:", uiFriendlyNumber(totalTime));
193             allScores.push(totalTime);
194         }
195
196         let allScores = [];
197         for (let benchmark of this.benchmarks)
198             allScores.push(benchmark.score);
199
200         if (isInBrowser) {
201             summaryElement.classList.add('done');
202             summaryElement.innerHTML = "<div class=\"score\">" + uiFriendlyNumber(geomean(allScores)) + "</div><label>Score</label>";
203             statusElement.innerHTML = '';
204         } else
205             console.log("\nTotal Score: ", uiFriendlyNumber(geomean(allScores)), "\n");
206
207         this.reportScoreToRunBenchmarkRunner();
208     }
209
210     runCode(string)
211     {
212         if (!isInBrowser) {
213             let scripts = string;
214             let globalObject = runString("");
215             globalObject.console = {log:globalObject.print}
216             globalObject.top = {
217                 currentResolve,
218                 currentReject
219             };
220             for (let script of scripts)
221                 globalObject.loadString(script);
222             return globalObject;
223         }
224
225         var magic = document.getElementById("magic");
226         magic.contentDocument.body.textContent = "";
227         magic.contentDocument.body.innerHTML = "<iframe id=\"magicframe\" frameborder=\"0\">";
228
229         var magicFrame = magic.contentDocument.getElementById("magicframe");
230         magicFrame.contentDocument.open();
231         magicFrame.contentDocument.write("<!DOCTYPE html><head><title>benchmark payload</title></head><body>\n" + string + "</body></html>");
232
233         return magicFrame;
234     }
235
236     prepareToRun()
237     {
238         this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
239
240         let text = "";
241         let newBenchmarks = [];
242         for (let benchmark of this.benchmarks) {
243             let id = JSON.stringify(benchmark.constructor.scoreDescription());
244             let description = JSON.parse(id);
245
246             newBenchmarks.push(benchmark);
247             let scoreIds = benchmark.scoreIdentifiers()
248             let overallScoreId = scoreIds.pop();
249
250             if (isInBrowser) {
251                 text +=
252                     `<div class="benchmark" id="benchmark-${benchmark.name}">
253                     <h3 class="benchmark-name"><a href="in-depth.html#${benchmark.name}">${benchmark.name}</a></h3>
254                     <h4 class="score" id="${overallScoreId}">___</h4><p>`;
255                 for (let i = 0; i < scoreIds.length; i++) {
256                     let id = scoreIds[i];
257                     let label = description[i];
258                     text += `<span class="result"><span id="${id}">___</span><label>${label}</label></span>`
259                 }
260                 text += `</p></div>`;
261             }
262         }
263
264         if (!isInBrowser)
265             return;
266
267         for (let f = 0; f < 5; f++)
268             text += `<div class="benchmark fill"></div>`;
269
270         let timestamp = Date.now();
271         document.getElementById('jetstreams').style.backgroundImage = `url('jetstreams.svg?${timestamp}')`;
272         let resultsTable = document.getElementById("results");
273         resultsTable.innerHTML = text;
274
275         document.getElementById("magic").textContent = "";
276         document.addEventListener('keypress', function (e) {
277             if (e.which === 13)
278                 JetStream.start();
279         });
280     }
281
282     reportError(benchmark)
283     {
284         for (let id of benchmark.scoreIdentifiers())
285             document.getElementById(id).innerHTML = "error";
286     }
287
288     async initialize() {
289         await this.fetchResources();
290         this.prepareToRun();
291         if (isInBrowser && window.location.search == '?report=true') {
292             setTimeout(() => this.start(), 4000);
293         }
294     }
295
296     async fetchResources() {
297         for (let benchmark of this.benchmarks)
298             await benchmark.fetchResources();
299
300         if (!isInBrowser)
301             return;
302
303         let statusElement = document.getElementById("status");
304         statusElement.classList.remove('loading');
305         statusElement.innerHTML = `<a href="javascript:JetStream.start()" class="button">Start Test</a>`;
306         statusElement.onclick = () => {
307             statusElement.onclick = null;
308             JetStream.start();
309             return false;
310         }
311     }
312
313     async reportScoreToRunBenchmarkRunner()
314     {
315         if (!isInBrowser)
316             return;
317
318         if (window.location.search !== '?report=true')
319             return;
320
321         let results = {};
322         for (let benchmark of this.benchmarks) {
323             const subResults = {}
324             const subTimes = benchmark.subTimes();
325             for (const name in subTimes) {
326                 subResults[name] = {"metrics": {"Time": {"current": [toTimeValue(subTimes[name])]}}};
327             }
328             results[benchmark.name] = {
329                 "metrics" : {
330                     "Score" : {"current" : [benchmark.score]},
331                     "Time": ["Geometric"],
332                 },
333                 "tests": subResults,
334             };;
335         }
336
337         results = {"JetStream2.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
338
339         const content = JSON.stringify(results);
340         await fetch("/report", {
341             method: "POST",
342             heeaders: {
343                 "Content-Type": "application/json",
344                 "Content-Length": content.length,
345                 "Connection": "close",
346             },
347             body: content,
348         });
349     }
350 };
351
352 class Benchmark {
353     constructor(plan)
354     {
355         this.plan = plan;
356         this.iterations = testIterationCount || plan.iterations || defaultIterationCount;
357         this.isAsync = !!plan.isAsync;
358
359         this.scripts = null;
360
361         this._resourcesPromise = null;
362         this.fetchResources();
363     }
364
365     get name() { return this.plan.name; }
366
367     get runnerCode() {
368         return `
369             let __benchmark = new Benchmark(${this.iterations});
370             let results = [];
371             for (let i = 0; i < ${this.iterations}; i++) {
372                 if (__benchmark.prepareForNextIteration)
373                     __benchmark.prepareForNextIteration();
374
375                 let start = Date.now();
376                 __benchmark.runIteration();
377                 let end = Date.now();
378
379                 results.push(Math.max(1, end - start));
380             }
381             if (__benchmark.validate)
382                 __benchmark.validate();
383             top.currentResolve(results);`;
384     }
385
386     processResults() {
387         throw new Error("Subclasses need to implement this");
388     }
389
390     get score() {
391         throw new Error("Subclasses need to implement this");
392     }
393
394     get prerunCode() { return null; }
395
396     async run() {
397         let code;
398         if (isInBrowser)
399             code = "";
400         else
401             code = [];
402
403         let addScript = (text) => {
404             if (isInBrowser)
405                 code += `<script>${text}</script>`;
406             else
407                 code.push(text);
408         };
409
410         let addScriptWithURL = (url) => {
411             if (isInBrowser)
412                 code += `<script src="${url}"></script>`;
413             else
414                 assert(false, "Should not reach here in CLI");
415         };
416
417         addScript(`const isInBrowser = ${isInBrowser}; let performance = {now: Date.now.bind(Date)};`);
418
419         if (!!this.plan.deterministicRandom) {
420             addScript(`
421                 Math.random = (function() {
422                     var seed = 49734321;
423                     return function() {
424                         // Robert Jenkins' 32 bit integer hash function.
425                         seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
426                         seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
427                         seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
428                         seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
429                         seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
430                         seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
431                         return (seed & 0xfffffff) / 0x10000000;
432                     };
433                 })();
434             `);
435
436         }
437
438         if (this.plan.preload) {
439             let str = "";
440             for (let [variableName, blobUrl] of this.preloads)
441                 str += `const ${variableName} = "${blobUrl}";\n`;
442             addScript(str);
443         }
444
445         let prerunCode = this.prerunCode;
446         if (prerunCode)
447             addScript(prerunCode);
448
449         if (preloadResources) {
450             assert(this.scripts && this.scripts.length === this.plan.files.length);
451
452             for (let text of this.scripts)
453                 addScript(text);
454         } else {
455             for (let file of this.plan.files)
456                 addScriptWithURL(file);
457         }
458
459         let promise = new Promise((resolve, reject) => {
460             currentResolve = resolve;
461             currentReject = reject;
462         });
463
464         if (isInBrowser) {
465             code = `
466                 <script> window.onerror = top.currentReject; </script>
467                 ${code}
468             `;
469         }
470         addScript(this.runnerCode);
471
472         this.startTime = new Date();
473
474         if (RAMification)
475             resetMemoryPeak();
476
477         let magicFrame;
478         try {
479             magicFrame = JetStream.runCode(code);
480         } catch(e) {
481             console.log("Error in runCode: ", e);
482             throw e;
483         }
484         let results = await promise;
485
486         this.endTime = new Date();
487
488         if (RAMification) {
489             let memoryFootprint = MemoryFootprint();
490             this.currentFootprint = memoryFootprint.current;
491             this.peakFootprint = memoryFootprint.peak;
492         }
493
494         this.processResults(results);
495         if (isInBrowser)
496             magicFrame.contentDocument.close();
497     }
498
499     fetchResources() {
500         if (this._resourcesPromise)
501             return this._resourcesPromise;
502
503         let filePromises = preloadResources ? this.plan.files.map((file) => fileLoader.load(file)) : [];
504         let preloads = [];
505         let preloadVariableNames = [];
506
507         if (isInBrowser && this.plan.preload) {
508             for (let prop of Object.getOwnPropertyNames(this.plan.preload)) {
509                 preloadVariableNames.push(prop);
510                 preloads.push(this.plan.preload[prop]);
511             }
512         }
513
514         preloads = preloads.map((file) => fileLoader.load(file));
515
516         let p1 = Promise.all(filePromises).then((texts) => {
517             if (!preloadResources)
518                 return;
519             this.scripts = [];
520             assert(texts.length === this.plan.files.length);
521             for (let text of texts)
522                 this.scripts.push(text);
523         });
524
525         let p2 = Promise.all(preloads).then((data) => {
526             this.preloads = [];
527             this.blobs = [];
528             for (let i = 0; i < data.length; ++i) {
529                 let item = data[i];
530
531                 let blob;
532                 if (typeof item === "string") {
533                     blob = new Blob([item], {type : 'application/javascript'});
534                 } else if (item instanceof ArrayBuffer) {
535                     blob = new Blob([item], {type : 'application/octet-stream'});
536                 } else
537                     throw new Error("Unexpected item!");
538
539                 this.blobs.push(blob);
540                 this.preloads.push([preloadVariableNames[i], URL.createObjectURL(blob)]);
541             }
542         });
543
544         this._resourcesPromise = Promise.all([p1, p2]);
545         return this._resourcesPromise;
546     }
547
548     static scoreDescription() { throw new Error("Must be implemented by subclasses."); }
549     scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); }
550
551     updateUIBeforeRun() {
552         if (!isInBrowser) {
553             console.log(`Running ${this.name}:`);
554             return;
555         }
556
557         let containerUI = document.getElementById("results");
558         let resultsBenchmarkUI = document.getElementById(`benchmark-${this.name}`);
559         containerUI.insertBefore(resultsBenchmarkUI, containerUI.firstChild);
560         resultsBenchmarkUI.classList.add("benchmark-running");
561
562         for (let id of this.scoreIdentifiers())
563             document.getElementById(id).innerHTML = "...";
564     }
565
566     updateUIAfterRun() {
567         if (!isInBrowser)
568             return;
569
570         let benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`);
571         benchmarkResultsUI.classList.remove("benchmark-running");
572         benchmarkResultsUI.classList.add("benchmark-done");
573
574     }
575 };
576
577 class DefaultBenchmark extends Benchmark {
578     constructor(...args) {
579         super(...args);
580
581         this.worstCaseCount = this.plan.worstCaseCount || defaultWorstCaseCount;
582         this.firstIteration = null;
583         this.worst4 = null;
584         this.average = null;
585     }
586
587     processResults(results) {
588         function copyArray(a) {
589             let result = [];
590             for (let x of a)
591                 result.push(x);
592             return result;
593         }
594         results = copyArray(results);
595
596         this.firstIteration = toScore(results[0]);
597
598         results = results.slice(1);
599         results.sort((a, b) => a < b ? 1 : -1);
600         for (let i = 0; i + 1 < results.length; ++i)
601             assert(results[i] >= results[i + 1]);
602
603         let worstCase = [];
604         for (let i = 0; i < this.worstCaseCount; ++i)
605             worstCase.push(results[i]);
606         this.worst4 = toScore(mean(worstCase));
607         this.average = toScore(mean(results));
608     }
609
610     get score() {
611         return geomean([this.firstIteration, this.worst4, this.average]);
612     }
613
614     subTimes() {
615         return {
616             "First": this.firstIteration,
617             "Worst": this.worst4,
618             "Average": this.average,
619         };
620     }
621
622     static scoreDescription() {
623         return ["First", "Worst", "Average", "Score"];
624     }
625
626     scoreIdentifiers() {
627         return [firstID(this), worst4ID(this), avgID(this), scoreID(this)];
628     }
629
630     updateUIAfterRun() {
631         super.updateUIAfterRun();
632
633         if (isInBrowser) {
634             document.getElementById(firstID(this)).innerHTML = uiFriendlyNumber(this.firstIteration);
635             document.getElementById(worst4ID(this)).innerHTML = uiFriendlyNumber(this.worst4);
636             document.getElementById(avgID(this)).innerHTML = uiFriendlyNumber(this.average);
637             document.getElementById(scoreID(this)).innerHTML = uiFriendlyNumber(this.score);
638             return;
639         }
640
641         print("    Startup:", uiFriendlyNumber(this.firstIteration));
642         print("    Worst Case:", uiFriendlyNumber(this.worst4));
643         print("    Average:", uiFriendlyNumber(this.average));
644         print("    Score:", uiFriendlyNumber(this.score));
645         if (RAMification) {
646             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
647             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
648         }
649         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
650     }
651 }
652
653 class AsyncBenchmark extends DefaultBenchmark {
654     get runnerCode() {
655         return `
656         async function doRun() {
657             let __benchmark = new Benchmark();
658             let results = [];
659             for (let i = 0; i < ${this.iterations}; i++) {
660                 let start = Date.now();
661                 await __benchmark.runIteration();
662                 let end = Date.now();
663                 results.push(Math.max(1, end - start));
664             }
665             if (__benchmark.validate)
666                 __benchmark.validate();
667             top.currentResolve(results);
668         }
669         doRun();`
670     }
671 };
672
673 class WSLBenchmark extends Benchmark {
674     constructor(...args) {
675         super(...args);
676
677         this.stdlib = null;
678         this.mainRun = null;
679     }
680
681     processResults(results) {
682         this.stdlib = toScore(results[0]);
683         this.mainRun = toScore(results[1]);
684     }
685
686     get score() {
687         return geomean([this.stdlib, this.mainRun]);
688     }
689
690     get runnerCode() {
691         return `
692             let benchmark = new Benchmark();
693             let results = [];
694             {
695                 let start = Date.now();
696                 benchmark.buildStdlib();
697                 results.push(Date.now() - start);
698             }
699
700             {
701                 let start = Date.now();
702                 benchmark.run();
703                 results.push(Date.now() - start);
704             }
705
706             top.currentResolve(results);
707             `;
708     }
709
710     subTimes() {
711         return {
712             "Stdlib": this.stdlib,
713             "MainRun": this.mainRun,
714         };
715     }
716
717     static scoreDescription() {
718         return ["Stdlib", "MainRun", "Score"];
719     }
720
721     scoreIdentifiers() {
722         return ["wsl-stdlib-score", "wsl-tests-score", "wsl-score-score"];
723     }
724
725     updateUIAfterRun() {
726         super.updateUIAfterRun();
727
728         if (isInBrowser) {
729             document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyNumber(this.stdlib);
730             document.getElementById("wsl-tests-score").innerHTML = uiFriendlyNumber(this.mainRun);
731             document.getElementById("wsl-score-score").innerHTML = uiFriendlyNumber(this.score);
732             return;
733         }
734
735         print("    Stdlib:", uiFriendlyNumber(this.stdlib));
736         print("    Tests:", uiFriendlyNumber(this.mainRun));
737         print("    Score:", uiFriendlyNumber(this.score));
738         if (RAMification) {
739             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
740             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
741         }
742         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
743     }
744 };
745
746 class WasmBenchmark extends Benchmark {
747     constructor(...args) {
748         super(...args);
749
750         this.startupTime = null;
751         this.runTime = null;
752     }
753
754     processResults(results) {
755         this.startupTime = toScore(results[0]);
756         this.runTime = toScore(results[1]);
757     }
758
759     get score() {
760         return geomean([this.startupTime, this.runTime]);
761     }
762
763     get wasmPath() {
764         return this.plan.wasmPath;
765     }
766
767     get prerunCode() {
768         let str = `
769             let verbose = false;
770
771             let compileTime = null;
772             let runTime = null;
773
774             let globalObject = this;
775
776             globalObject.benchmarkTime = Date.now.bind(Date);
777
778             globalObject.reportCompileTime = (t) => {
779                 if (compileTime !== null)
780                     throw new Error("called report compile time twice");
781                 compileTime = t;
782             };
783
784             globalObject.reportRunTime = (t) => {
785                 if (runTime !== null)
786                     throw new Error("called report run time twice")
787                 runTime = t;
788                 top.currentResolve([compileTime, runTime]);
789             };
790
791             abort = quit = function() {
792                 if (verbose)
793                     console.log('Intercepted quit/abort');
794             };
795
796             oldPrint = globalObject.print;
797             globalObject.print = globalObject.printErr = (...args) => {
798                 if (verbose)
799                     console.log('Intercepted print: ', ...args);
800             };
801
802             let Module = {
803                 preRun: [],
804                 postRun: [],
805                 print: function() { },
806                 printErr: function() { },
807                 setStatus: function(text) {
808                 },
809                 totalDependencies: 0,
810                 monitorRunDependencies: function(left) {
811                     this.totalDependencies = Math.max(this.totalDependencies, left);
812                     Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
813                 }
814             };
815             globalObject.Module = Module;
816             `;
817         return str;
818     }
819
820     get runnerCode() {
821         let str = "";
822         if (isInBrowser) {
823             str += `
824                 var xhr = new XMLHttpRequest();
825                 xhr.open('GET', wasmBlobURL, true);
826                 xhr.responseType = 'arraybuffer';
827                 xhr.onload = function() {
828                     Module.wasmBinary = xhr.response;
829                     doRun();
830                 };
831                 xhr.send(null);
832             `;
833         } else {
834             str += `
835             Module.wasmBinary = read("${this.wasmPath}", "binary");
836             globalObject.read = (...args) => {
837                 console.log("should not be inside read: ", ...args);
838                 throw new Error;
839             };
840
841             Module.setStatus = null;
842             Module.monitorRunDependencies = null;
843
844             Promise.resolve(42).then(() => {
845                 try {
846                     doRun();
847                 } catch(e) {
848                     console.log("error running wasm:", e);
849                     throw e;
850                 }
851             })
852             `;
853         }
854         return str;
855     }
856
857     subTimes() {
858         return {
859             "Startup": this.startupTime,
860             "Runtime": this.runTime,
861         };
862     }
863
864     static scoreDescription() {
865         return ["Startup", "Runtime", "Score"];
866     }
867
868     get startupID() {
869         return `wasm-startup-id${this.name}`;
870     }
871     get runID() {
872         return `wasm-run-id${this.name}`;
873     }
874     get scoreID() {
875         return `wasm-score-id${this.name}`;
876     }
877
878     scoreIdentifiers() {
879         return [this.startupID, this.runID, this.scoreID];
880     }
881
882     updateUIAfterRun() {
883         super.updateUIAfterRun();
884
885         if (isInBrowser) {
886             document.getElementById(this.startupID).innerHTML = uiFriendlyNumber(this.startupTime);
887             document.getElementById(this.runID).innerHTML = uiFriendlyNumber(this.runTime);
888             document.getElementById(this.scoreID).innerHTML = uiFriendlyNumber(this.score);
889             return;
890         }
891         print("    Startup:", uiFriendlyNumber(this.startupTime));
892         print("    Run time:", uiFriendlyNumber(this.runTime));
893         if (RAMification) {
894             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
895             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
896         }
897         print("    Score:", uiFriendlyNumber(this.score));
898     }
899 };
900
901 const ARESGroup = Symbol.for("ARES");
902 const CDJSGroup = Symbol.for("CDJS");
903 const CodeLoadGroup = Symbol.for("CodeLoad");
904 const LuaJSFightGroup = Symbol.for("LuaJSFight");
905 const OctaneGroup = Symbol.for("Octane");
906 const RexBenchGroup = Symbol.for("RexBench");
907 const SeaMonsterGroup = Symbol.for("SeaMonster");
908 const SimpleGroup = Symbol.for("Simple");
909 const SunSpiderGroup = Symbol.for("SunSpider");
910 const WasmGroup = Symbol.for("Wasm");
911 const WorkerTestsGroup = Symbol.for("WorkerTests");
912 const WSLGroup = Symbol.for("WSL");
913 const WTBGroup = Symbol.for("WTB");
914
915
916 let testPlans = [
917     // ARES
918     {
919         name: "Air",
920         files: [
921             "./ARES-6/Air/symbols.js"
922             , "./ARES-6/Air/tmp_base.js"
923             , "./ARES-6/Air/arg.js"
924             , "./ARES-6/Air/basic_block.js"
925             , "./ARES-6/Air/code.js"
926             , "./ARES-6/Air/frequented_block.js"
927             , "./ARES-6/Air/inst.js"
928             , "./ARES-6/Air/opcode.js"
929             , "./ARES-6/Air/reg.js"
930             , "./ARES-6/Air/stack_slot.js"
931             , "./ARES-6/Air/tmp.js"
932             , "./ARES-6/Air/util.js"
933             , "./ARES-6/Air/custom.js"
934             , "./ARES-6/Air/liveness.js"
935             , "./ARES-6/Air/insertion_set.js"
936             , "./ARES-6/Air/allocate_stack.js"
937             , "./ARES-6/Air/payload-gbemu-executeIteration.js"
938             , "./ARES-6/Air/payload-imaging-gaussian-blur-gaussianBlur.js"
939             , "./ARES-6/Air/payload-airjs-ACLj8C.js"
940             , "./ARES-6/Air/payload-typescript-scanIdentifier.js"
941             , "./ARES-6/Air/benchmark.js"
942         ],
943         testGroup: ARESGroup
944     },
945     {
946         name: "Basic",
947         files: [
948             "./ARES-6/Basic/ast.js"
949             , "./ARES-6/Basic/basic.js"
950             , "./ARES-6/Basic/caseless_map.js"
951             , "./ARES-6/Basic/lexer.js"
952             , "./ARES-6/Basic/number.js"
953             , "./ARES-6/Basic/parser.js"
954             , "./ARES-6/Basic/random.js"
955             , "./ARES-6/Basic/state.js"
956             , "./ARES-6/Basic/util.js"
957             , "./ARES-6/Basic/benchmark.js"
958         ],
959         testGroup: ARESGroup
960     },
961     {
962         name: "ML",
963         files: [
964             "./ARES-6/ml/index.js"
965             , "./ARES-6/ml/benchmark.js"
966         ],
967         iterations: 60,
968         testGroup: ARESGroup
969     },
970     {
971         name: "Babylon",
972         files: [
973             "./ARES-6/Babylon/index.js"
974             , "./ARES-6/Babylon/benchmark.js"
975         ],
976         preload: {
977             airBlob: "./ARES-6/Babylon/air-blob.js",
978             basicBlob: "./ARES-6/Babylon/basic-blob.js",
979             inspectorBlob: "./ARES-6/Babylon/inspector-blob.js",
980             babylonBlob: "./ARES-6/Babylon/babylon-blob.js"
981         },
982         testGroup: ARESGroup
983     },
984     // CDJS
985     {
986         name: "cdjs",
987         files: [
988             "./cdjs/constants.js"
989             , "./cdjs/util.js"
990             , "./cdjs/red_black_tree.js"
991             , "./cdjs/call_sign.js"
992             , "./cdjs/vector_2d.js"
993             , "./cdjs/vector_3d.js"
994             , "./cdjs/motion.js"
995             , "./cdjs/reduce_collision_set.js"
996             , "./cdjs/simulator.js"
997             , "./cdjs/collision.js"
998             , "./cdjs/collision_detector.js"
999             , "./cdjs/benchmark.js"
1000         ],
1001         iterations: 60,
1002         worstCaseCount: 3,
1003         testGroup: CDJSGroup
1004     },
1005     // CodeLoad
1006     {
1007         name: "first-inspector-code-load",
1008         files: [
1009             "./code-load/code-first-load.js"
1010         ],
1011         preload: {
1012             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1013         },
1014         testGroup: CodeLoadGroup
1015     },
1016     {
1017         name: "multi-inspector-code-load",
1018         files: [
1019             "./code-load/code-multi-load.js"
1020         ],
1021         preload: {
1022             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1023         },
1024         testGroup: CodeLoadGroup
1025     },
1026     // Octane
1027     {
1028         name: "Box2D",
1029         files: [
1030             "./Octane/box2d.js"
1031         ],
1032         deterministicRandom: true,
1033         testGroup: OctaneGroup
1034     },
1035     {
1036         name: "octane-code-load",
1037         files: [
1038             "./Octane/code-first-load.js"
1039         ],
1040         deterministicRandom: true,
1041         testGroup: OctaneGroup
1042     },
1043     {
1044         name: "crypto",
1045         files: [
1046             "./Octane/crypto.js"
1047         ],
1048         deterministicRandom: true,
1049         testGroup: OctaneGroup
1050     },
1051     {
1052         name: "delta-blue",
1053         files: [
1054             "./Octane/deltablue.js"
1055         ],
1056         deterministicRandom: true,
1057         testGroup: OctaneGroup
1058     },
1059     {
1060         name: "earley-boyer",
1061         files: [
1062             "./Octane/earley-boyer.js"
1063         ],
1064         deterministicRandom: true,
1065         testGroup: OctaneGroup
1066     },
1067     {
1068         name: "gbemu",
1069         files: [
1070             "./Octane/gbemu-part1.js"
1071             , "./Octane/gbemu-part2.js"
1072         ],
1073         deterministicRandom: true,
1074         testGroup: OctaneGroup
1075     },
1076     {
1077         name: "mandreel",
1078         files: [
1079             "./Octane/mandreel.js"
1080         ],
1081         iterations: 80,
1082         deterministicRandom: true,
1083         testGroup: OctaneGroup
1084     },
1085     {
1086         name: "navier-stokes",
1087         files: [
1088             "./Octane/navier-stokes.js"
1089         ],
1090         deterministicRandom: true,
1091         testGroup: OctaneGroup
1092     },
1093     {
1094         name: "pdfjs",
1095         files: [
1096             "./Octane/pdfjs.js"
1097         ],
1098         deterministicRandom: true,
1099         testGroup: OctaneGroup
1100     },
1101     {
1102         name: "raytrace",
1103         files: [
1104             "./Octane/raytrace.js"
1105         ],
1106         deterministicRandom: true,
1107         testGroup: OctaneGroup
1108     },
1109     {
1110         name: "regexp",
1111         files: [
1112             "./Octane/regexp.js"
1113         ],
1114         deterministicRandom: true,
1115         testGroup: OctaneGroup
1116     },
1117     {
1118         name: "richards",
1119         files: [
1120             "./Octane/richards.js"
1121         ],
1122         deterministicRandom: true,
1123         testGroup: OctaneGroup
1124     },
1125     {
1126         name: "splay",
1127         files: [
1128             "./Octane/splay.js"
1129         ],
1130         deterministicRandom: true,
1131         testGroup: OctaneGroup
1132     },
1133     {
1134         name: "typescript",
1135         files: [
1136             "./Octane/typescript-compiler.js"
1137             , "./Octane/typescript-input.js"
1138             , "./Octane/typescript.js"
1139         ],
1140         iterations: 15,
1141         worstCaseCount: 2,
1142         deterministicRandom: true,
1143         testGroup: OctaneGroup
1144     },
1145     {
1146         name: "octane-zlib",
1147         files: [
1148             "./Octane/zlib-data.js"
1149             , "./Octane/zlib.js"
1150         ],
1151         iterations: 15,
1152         worstCaseCount: 2,
1153         deterministicRandom: true,
1154         testGroup: OctaneGroup
1155     },
1156     // RexBench
1157     {
1158         name: "FlightPlanner",
1159         files: [
1160             "./RexBench/FlightPlanner/airways.js"
1161             , "./RexBench/FlightPlanner/waypoints.js"
1162             , "./RexBench/FlightPlanner/flight_planner.js"
1163             , "./RexBench/FlightPlanner/expectations.js"
1164             , "./RexBench/FlightPlanner/benchmark.js"
1165         ],
1166         testGroup: RexBenchGroup
1167     },
1168     {
1169         name: "OfflineAssembler",
1170         files: [
1171             "./RexBench/OfflineAssembler/registers.js"
1172             , "./RexBench/OfflineAssembler/instructions.js"
1173             , "./RexBench/OfflineAssembler/ast.js"
1174             , "./RexBench/OfflineAssembler/parser.js"
1175             , "./RexBench/OfflineAssembler/file.js"
1176             , "./RexBench/OfflineAssembler/LowLevelInterpreter.js"
1177             , "./RexBench/OfflineAssembler/LowLevelInterpreter32_64.js"
1178             , "./RexBench/OfflineAssembler/LowLevelInterpreter64.js"
1179             , "./RexBench/OfflineAssembler/InitBytecodes.js"
1180             , "./RexBench/OfflineAssembler/expected.js"
1181             , "./RexBench/OfflineAssembler/benchmark.js"
1182         ],
1183         iterations: 80,
1184         testGroup: RexBenchGroup
1185     },
1186     {
1187         name: "UniPoker",
1188         files: [
1189             "./RexBench/UniPoker/poker.js"
1190             , "./RexBench/UniPoker/expected.js"
1191             , "./RexBench/UniPoker/benchmark.js"
1192         ],
1193         deterministicRandom: true,
1194         testGroup: RexBenchGroup
1195     },
1196     // Simple
1197     {
1198         name: "async-fs",
1199         files: [
1200             "./simple/file-system.js"
1201         ],
1202         iterations: 40,
1203         worstCaseCount: 3,
1204         benchmarkClass: AsyncBenchmark,
1205         testGroup: SimpleGroup
1206     },
1207     {
1208         name: "float-mm.c",
1209         files: [
1210             "./simple/float-mm.c.js"
1211         ],
1212         iterations: 15,
1213         worstCaseCount: 2,
1214         testGroup: SimpleGroup
1215     },
1216     {
1217         name: "hash-map",
1218         files: [
1219             "./simple/hash-map.js"
1220         ],
1221         testGroup: SimpleGroup
1222     },
1223     // SeaMonster
1224     {
1225         name: "ai-astar",
1226         files: [
1227             "./SeaMonster/ai-astar.js"
1228         ],
1229         testGroup: SeaMonsterGroup
1230     },
1231     {
1232         name: "gaussian-blur",
1233         files: [
1234             "./SeaMonster/gaussian-blur.js"
1235         ],
1236         testGroup: SeaMonsterGroup
1237     },
1238     {
1239         name: "stanford-crypto-aes",
1240         files: [
1241             "./SeaMonster/sjlc.js"
1242             , "./SeaMonster/stanford-crypto-aes.js"
1243         ],
1244         testGroup: SeaMonsterGroup
1245     },
1246     {
1247         name: "stanford-crypto-pbkdf2",
1248         files: [
1249             "./SeaMonster/sjlc.js"
1250             , "./SeaMonster/stanford-crypto-pbkdf2.js"
1251         ],
1252         testGroup: SeaMonsterGroup
1253     },
1254     {
1255         name: "stanford-crypto-sha256",
1256         files: [
1257             "./SeaMonster/sjlc.js"
1258             , "./SeaMonster/stanford-crypto-sha256.js"
1259         ],
1260         testGroup: SeaMonsterGroup
1261     },
1262     {
1263         name: "json-stringify-inspector",
1264         files: [
1265             "./SeaMonster/inspector-json-payload.js"
1266             , "./SeaMonster/json-stringify-inspector.js"
1267         ],
1268         iterations: 20,
1269         worstCaseCount: 2,
1270         testGroup: SeaMonsterGroup
1271     },
1272     {
1273         name: "json-parse-inspector",
1274         files: [
1275             "./SeaMonster/inspector-json-payload.js"
1276             , "./SeaMonster/json-parse-inspector.js"
1277         ],
1278         iterations: 20,
1279         worstCaseCount: 2,
1280         testGroup: SeaMonsterGroup
1281     },
1282     // Wasm
1283     {
1284         name: "HashSet-wasm",
1285         wasmPath: "./wasm/HashSet.wasm",
1286         files: [
1287             "./wasm/HashSet.js"
1288         ],
1289         preload: {
1290             wasmBlobURL: "./wasm/HashSet.wasm"
1291         },
1292         benchmarkClass: WasmBenchmark,
1293         testGroup: WasmGroup
1294     },
1295     {
1296         name: "tsf-wasm",
1297         wasmPath: "./wasm/tsf.wasm",
1298         files: [
1299             "./wasm/tsf.js"
1300         ],
1301         preload: {
1302             wasmBlobURL: "./wasm/tsf.wasm"
1303         },
1304         benchmarkClass: WasmBenchmark,
1305         testGroup: WasmGroup
1306     },
1307     {
1308         name: "quicksort-wasm",
1309         wasmPath: "./wasm/quicksort.wasm",
1310         files: [
1311             "./wasm/quicksort.js"
1312         ],
1313         preload: {
1314             wasmBlobURL: "./wasm/quicksort.wasm"
1315         },
1316         benchmarkClass: WasmBenchmark,
1317         testGroup: WasmGroup
1318     },
1319     {
1320         name: "gcc-loops-wasm",
1321         wasmPath: "./wasm/gcc-loops.wasm",
1322         files: [
1323             "./wasm/gcc-loops.js"
1324         ],
1325         preload: {
1326             wasmBlobURL: "./wasm/gcc-loops.wasm"
1327         },
1328         benchmarkClass: WasmBenchmark,
1329         testGroup: WasmGroup
1330     },
1331     {
1332         name: "richards-wasm",
1333         wasmPath: "./wasm/richards.wasm",
1334         files: [
1335             "./wasm/richards.js"
1336         ],
1337         preload: {
1338             wasmBlobURL: "./wasm/richards.wasm"
1339         },
1340         benchmarkClass: WasmBenchmark,
1341         testGroup: WasmGroup
1342     },
1343     // WorkerTests
1344     {
1345         name: "bomb-workers",
1346         files: [
1347             "./worker/bomb.js"
1348         ],
1349         iterations: 80,
1350         preload: {
1351             rayTrace3D: "./worker/bomb-subtests/3d-raytrace.js"
1352             , accessNbody: "./worker/bomb-subtests/access-nbody.js"
1353             , morph3D: "./worker/bomb-subtests/3d-morph.js"
1354             , cube3D: "./worker/bomb-subtests/3d-cube.js"
1355             , accessFunnkuch: "./worker/bomb-subtests/access-fannkuch.js"
1356             , accessBinaryTrees: "./worker/bomb-subtests/access-binary-trees.js"
1357             , accessNsieve: "./worker/bomb-subtests/access-nsieve.js"
1358             , bitopsBitwiseAnd: "./worker/bomb-subtests/bitops-bitwise-and.js"
1359             , bitopsNsieveBits: "./worker/bomb-subtests/bitops-nsieve-bits.js"
1360             , controlflowRecursive: "./worker/bomb-subtests/controlflow-recursive.js"
1361             , bitops3BitBitsInByte: "./worker/bomb-subtests/bitops-3bit-bits-in-byte.js"
1362             , botopsBitsInByte: "./worker/bomb-subtests/bitops-bits-in-byte.js"
1363             , cryptoAES: "./worker/bomb-subtests/crypto-aes.js"
1364             , cryptoMD5: "./worker/bomb-subtests/crypto-md5.js"
1365             , cryptoSHA1: "./worker/bomb-subtests/crypto-sha1.js"
1366             , dateFormatTofte: "./worker/bomb-subtests/date-format-tofte.js"
1367             , dateFormatXparb: "./worker/bomb-subtests/date-format-xparb.js"
1368             , mathCordic: "./worker/bomb-subtests/math-cordic.js"
1369             , mathPartialSums: "./worker/bomb-subtests/math-partial-sums.js"
1370             , mathSpectralNorm: "./worker/bomb-subtests/math-spectral-norm.js"
1371             , stringBase64: "./worker/bomb-subtests/string-base64.js"
1372             , stringFasta: "./worker/bomb-subtests/string-fasta.js"
1373             , stringValidateInput: "./worker/bomb-subtests/string-validate-input.js"
1374             , stringTagcloud: "./worker/bomb-subtests/string-tagcloud.js"
1375             , stringUnpackCode: "./worker/bomb-subtests/string-unpack-code.js"
1376             , regexpDNA: "./worker/bomb-subtests/regexp-dna.js"
1377         },
1378         benchmarkClass: AsyncBenchmark,
1379         testGroup: WorkerTestsGroup
1380     },
1381     {
1382         name: "segmentation",
1383         files: [
1384             "./worker/segmentation.js"
1385         ],
1386         preload: {
1387             asyncTaskBlob: "./worker/async-task.js"
1388         },
1389         iterations: 36,
1390         worstCaseCount: 3,
1391         benchmarkClass: AsyncBenchmark,
1392         testGroup: WorkerTestsGroup
1393     },
1394     // WSL
1395     {
1396         name: "WSL",
1397         files: ["./WSL/Node.js" ,"./WSL/Type.js" ,"./WSL/ReferenceType.js" ,"./WSL/Value.js" ,"./WSL/Expression.js" ,"./WSL/Rewriter.js" ,"./WSL/Visitor.js" ,"./WSL/CreateLiteral.js" ,"./WSL/CreateLiteralType.js" ,"./WSL/PropertyAccessExpression.js" ,"./WSL/AddressSpace.js" ,"./WSL/AnonymousVariable.js" ,"./WSL/ArrayRefType.js" ,"./WSL/ArrayType.js" ,"./WSL/Assignment.js" ,"./WSL/AutoWrapper.js" ,"./WSL/Block.js" ,"./WSL/BoolLiteral.js" ,"./WSL/Break.js" ,"./WSL/CallExpression.js" ,"./WSL/CallFunction.js" ,"./WSL/Check.js" ,"./WSL/CheckLiteralTypes.js" ,"./WSL/CheckLoops.js" ,"./WSL/CheckRecursiveTypes.js" ,"./WSL/CheckRecursion.js" ,"./WSL/CheckReturns.js" ,"./WSL/CheckUnreachableCode.js" ,"./WSL/CheckWrapped.js" ,"./WSL/Checker.js" ,"./WSL/CloneProgram.js" ,"./WSL/CommaExpression.js" ,"./WSL/ConstexprFolder.js" ,"./WSL/ConstexprTypeParameter.js" ,"./WSL/Continue.js" ,"./WSL/ConvertPtrToArrayRefExpression.js" ,"./WSL/DereferenceExpression.js" ,"./WSL/DoWhileLoop.js" ,"./WSL/DotExpression.js" ,"./WSL/DoubleLiteral.js" ,"./WSL/DoubleLiteralType.js" ,"./WSL/EArrayRef.js" ,"./WSL/EBuffer.js" ,"./WSL/EBufferBuilder.js" ,"./WSL/EPtr.js" ,"./WSL/EnumLiteral.js" ,"./WSL/EnumMember.js" ,"./WSL/EnumType.js" ,"./WSL/EvaluationCommon.js" ,"./WSL/Evaluator.js" ,"./WSL/ExpressionFinder.js" ,"./WSL/ExternalOrigin.js" ,"./WSL/Field.js" ,"./WSL/FindHighZombies.js" ,"./WSL/FlattenProtocolExtends.js" ,"./WSL/FlattenedStructOffsetGatherer.js" ,"./WSL/FloatLiteral.js" ,"./WSL/FloatLiteralType.js" ,"./WSL/FoldConstexprs.js" ,"./WSL/ForLoop.js" ,"./WSL/Func.js" ,"./WSL/FuncDef.js" ,"./WSL/FuncInstantiator.js" ,"./WSL/FuncParameter.js" ,"./WSL/FunctionLikeBlock.js" ,"./WSL/HighZombieFinder.js" ,"./WSL/IdentityExpression.js" ,"./WSL/IfStatement.js" ,"./WSL/IndexExpression.js" ,"./WSL/InferTypesForCall.js" ,"./WSL/Inline.js" ,"./WSL/Inliner.js" ,"./WSL/InstantiateImmediates.js" ,"./WSL/IntLiteral.js" ,"./WSL/IntLiteralType.js" ,"./WSL/Intrinsics.js" ,"./WSL/LateChecker.js" ,"./WSL/Lexer.js" ,"./WSL/LexerToken.js" ,"./WSL/LiteralTypeChecker.js" ,"./WSL/LogicalExpression.js" ,"./WSL/LogicalNot.js" ,"./WSL/LoopChecker.js" ,"./WSL/MakeArrayRefExpression.js" ,"./WSL/MakePtrExpression.js" ,"./WSL/NameContext.js" ,"./WSL/NameFinder.js" ,"./WSL/NameResolver.js" ,"./WSL/NativeFunc.js" ,"./WSL/NativeFuncInstance.js" ,"./WSL/NativeType.js" ,"./WSL/NativeTypeInstance.js" ,"./WSL/NormalUsePropertyResolver.js" ,"./WSL/NullLiteral.js" ,"./WSL/NullType.js" ,"./WSL/OriginKind.js" ,"./WSL/OverloadResolutionFailure.js" ,"./WSL/Parse.js" ,"./WSL/Prepare.js" ,"./WSL/Program.js" ,"./WSL/ProgramWithUnnecessaryThingsRemoved.js" ,"./WSL/PropertyResolver.js" ,"./WSL/Protocol.js" ,"./WSL/ProtocolDecl.js" ,"./WSL/ProtocolFuncDecl.js" ,"./WSL/ProtocolRef.js" ,"./WSL/PtrType.js" ,"./WSL/ReadModifyWriteExpression.js" ,"./WSL/RecursionChecker.js" ,"./WSL/RecursiveTypeChecker.js" ,"./WSL/ResolveNames.js" ,"./WSL/ResolveOverloadImpl.js" ,"./WSL/ResolveProperties.js" ,"./WSL/ResolveTypeDefs.js" ,"./WSL/Return.js" ,"./WSL/ReturnChecker.js" ,"./WSL/ReturnException.js" ,"./WSL/StandardLibrary.js" ,"./WSL/StatementCloner.js" ,"./WSL/StructLayoutBuilder.js" ,"./WSL/StructType.js" ,"./WSL/Substitution.js" ,"./WSL/SwitchCase.js" ,"./WSL/SwitchStatement.js" ,"./WSL/SynthesizeEnumFunctions.js" ,"./WSL/SynthesizeStructAccessors.js" ,"./WSL/TrapStatement.js" ,"./WSL/TypeDef.js" ,"./WSL/TypeDefResolver.js" ,"./WSL/TypeOrVariableRef.js" ,"./WSL/TypeParameterRewriter.js" ,"./WSL/TypeRef.js" ,"./WSL/TypeVariable.js" ,"./WSL/TypeVariableTracker.js" ,"./WSL/TypedValue.js" ,"./WSL/UintLiteral.js" ,"./WSL/UintLiteralType.js" ,"./WSL/UnificationContext.js" ,"./WSL/UnreachableCodeChecker.js" ,"./WSL/VariableDecl.js" ,"./WSL/VariableRef.js" ,"./WSL/VisitingSet.js" ,"./WSL/WSyntaxError.js" ,"./WSL/WTrapError.js" ,"./WSL/WTypeError.js" ,"./WSL/WhileLoop.js" ,"./WSL/WrapChecker.js", "./WSL/Test.js"],
1398         benchmarkClass: WSLBenchmark,
1399         testGroup: WSLGroup
1400     }
1401 ];
1402
1403 // LuaJSFight tests
1404 let luaJSFightTests = [
1405     "hello_world"
1406     , "list_search"
1407     , "lists"
1408     , "string_lists"
1409 ];
1410 for (let test of luaJSFightTests) {
1411     testPlans.push({
1412         name: `${test}-LJF`,
1413         files: [
1414             `./LuaJSFight/${test}.js`
1415         ],
1416         testGroup: LuaJSFightGroup
1417     });
1418 }
1419
1420 // SunSpider tests
1421 let sunSpiderTests = [
1422     "3d-cube"
1423     , "3d-raytrace"
1424     , "base64"
1425     , "crypto-aes"
1426     , "crypto-md5"
1427     , "crypto-sha1"
1428     , "date-format-tofte"
1429     , "date-format-xparb"
1430     , "n-body"
1431     , "regex-dna"
1432     , "string-unpack-code"
1433     , "tagcloud"
1434 ];
1435 for (let test of sunSpiderTests) {
1436     testPlans.push({
1437         name: `${test}-SP`,
1438         files: [
1439             `./SunSpider/${test}.js`
1440         ],
1441         testGroup: SunSpiderGroup
1442     });
1443 }
1444
1445 // WTB (Web Tooling Benchmark) tests
1446 let WTBTests = [
1447     "acorn"
1448     , "babylon"
1449     , "chai"
1450     , "coffeescript"
1451     , "espree"
1452     , "jshint"
1453     , "lebab"
1454     , "prepack"
1455     , "uglify-js"
1456 ];
1457 for (let name of WTBTests) {
1458     testPlans.push({
1459         name: `${name}-wtb`,
1460         files: [
1461             isInBrowser ? "./web-tooling-benchmark/browser.js" : "./web-tooling-benchmark/cli.js"
1462             , `./web-tooling-benchmark/${name}.js`
1463         ],
1464         iterations: 5,
1465         worstCaseCount: 1,
1466         testGroup: WTBGroup
1467     });
1468 }
1469
1470
1471 let testsByName = new Map();
1472 let testsByGroup = new Map();
1473
1474 for (let plan of testPlans) {
1475     let testName = plan.name;
1476
1477     if (testsByName.has(plan.name))
1478         throw "Duplicate test plan with name \"" + testName + "\"";
1479     else
1480         testsByName.set(testName, plan);
1481
1482     let group = plan.testGroup;
1483
1484     if (testsByGroup.has(group))
1485         testsByGroup.get(group).push(testName);
1486     else
1487         testsByGroup.set(group, [testName]);
1488 }
1489
1490 this.JetStream = new Driver();
1491
1492 function addTestByName(testName)
1493 {
1494     let plan = testsByName.get(testName);
1495
1496     if (plan)
1497         JetStream.addPlan(plan, plan.benchmarkClass);
1498     else
1499         throw "Couldn't find test named \"" +  testName + "\"";
1500 }
1501
1502 function addTestsByGroup(group)
1503 {
1504     let testList = testsByGroup.get(group);
1505
1506     if (!testList)
1507         throw "Couldn't find test group named: \"" + Symbol.keyFor(group) + "\"";
1508
1509     for (let testName of testList)
1510         addTestByName(testName);
1511 }
1512
1513 function processTestList(testList)
1514 {
1515     let tests = [];
1516
1517     if (testList instanceof Array)
1518         tests = testList;
1519     else
1520         tests = testList.split(/[\s,]/);
1521
1522     for (let testName of tests) {
1523         let groupTest = testsByGroup.get(Symbol.for(testName));
1524
1525         if (groupTest) {
1526             for (let testName of groupTest)
1527                 addTestByName(testName);
1528         } else
1529             addTestByName(testName);
1530     }
1531 }
1532
1533 let runOctane = true;
1534 let runARES = true;
1535 let runWSL = true;
1536 let runRexBench = true;
1537 let runWTB = true;
1538 let runSunSpider = true;
1539 let runSimple = true;
1540 let runCDJS = true;
1541 let runWorkerTests = !!isInBrowser;
1542 let runSeaMonster = true;
1543 let runCodeLoad = true;
1544 let runWasm = true;
1545
1546 if (false) {
1547     runOctane = false;
1548     runARES = false;
1549     runWSL = false;
1550     runRexBench = false;
1551     runWTB = false;
1552     runSunSpider = false;
1553     runSimple = false;
1554     runCDJS = false;
1555     runWorkerTests = false;
1556     runSeaMonster = false;
1557     runCodeLoad = false;
1558     runWasm = false;
1559 }
1560
1561 if (typeof testList !== "undefined") {
1562     processTestList(testList);
1563 } else {
1564     if (runARES)
1565         addTestsByGroup(ARESGroup);
1566
1567     if (runCDJS)
1568         addTestsByGroup(CDJSGroup);
1569
1570     if (runCodeLoad)
1571         addTestsByGroup(CodeLoadGroup);
1572
1573     if (runOctane)
1574         addTestsByGroup(OctaneGroup);
1575
1576     if (runRexBench)
1577         addTestsByGroup(RexBenchGroup);
1578
1579     if (runSeaMonster)
1580         addTestsByGroup(SeaMonsterGroup);
1581
1582     if (runSimple)
1583         addTestsByGroup(SimpleGroup);
1584
1585     if (runSunSpider)
1586         addTestsByGroup(SunSpiderGroup);
1587
1588     if (runWasm)
1589         addTestsByGroup(WasmGroup);
1590
1591     if (runWorkerTests)
1592         addTestsByGroup(WorkerTestsGroup);
1593
1594     if (runWSL)
1595         addTestsByGroup(WSLGroup);
1596
1597     if (runWTB)
1598         addTestsByGroup(WTBGroup);
1599 }