JS2 should print scores for different categories
[WebKit-https.git] / PerformanceTests / JetStream2 / JetStreamDriver.js
1 "use strict";
2
3 /*
4  * Copyright (C) 2018 Apple Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
19  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
25  * THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 const preloadResources = !isInBrowser;
29 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
30
31 if (typeof RAMification === "undefined")
32     var RAMification = false;
33
34 if (typeof testIterationCount === "undefined")
35     var testIterationCount = undefined;
36
37 // Used for the promise representing the current benchmark run.
38 this.currentResolve = null;
39 this.currentReject = null;
40
41 const defaultIterationCount = 120;
42 const defaultWorstCaseCount = 4;
43
44 let showScoreDetails = false;
45 let categoryScores = null;
46
47 function displayCategoryScores() {
48     if (!categoryScores)
49         return;
50
51     let summaryElement = document.getElementById("result-summary");
52     for (let [category, scores] of categoryScores)
53         summaryElement.innerHTML += `<p> ${category}: ${uiFriendlyNumber(geomean(scores))}</p>`
54
55     categoryScores = null;
56 }
57
58 if (isInBrowser) {
59     document.onkeydown = (keyboardEvent) => {
60         let key = keyboardEvent.key;
61         if (key === "d" || key === "D") {
62             showScoreDetails = true;
63
64             displayCategoryScores();
65         }
66     };
67 }
68
69 function assert(b, m = "") {
70     if (!b)
71         throw new Error("Bad assertion: " + m);
72 }
73
74 function firstID(benchmark) {
75     return `results-cell-${benchmark.name}-first`;
76 }
77
78 function worst4ID(benchmark) {
79     return `results-cell-${benchmark.name}-worst4`;
80 }
81
82 function avgID(benchmark) {
83     return `results-cell-${benchmark.name}-avg`;
84 }
85
86 function scoreID(benchmark) {
87     return `results-cell-${benchmark.name}-score`;
88 }
89
90 function mean(values) {
91     assert(values instanceof Array);
92     let sum = 0;
93     for (let x of values)
94         sum += x;
95     return sum / values.length;
96 }
97
98 function geomean(values) {
99     assert(values instanceof Array);
100     let product = 1;
101     for (let x of values)
102         product *= x;
103     return product ** (1 / values.length);
104 }
105
106 function toScore(timeValue) {
107     return 5000 / timeValue;
108 }
109
110 function toTimeValue(score) {
111     return 5000 / score;
112 }
113
114 function updateUI() {
115     return new Promise((resolve) => {
116         if (isInBrowser)
117             requestAnimationFrame(() => setTimeout(resolve, 0));
118         else
119             resolve();
120     });
121 }
122
123 function uiFriendlyNumber(num) {
124     if (Number.isInteger(num))
125         return num;
126     return num.toFixed(3);
127 }
128
129 function uiFriendlyDuration(time)
130 {
131     let minutes = time.getMinutes();
132     let seconds = time.getSeconds();
133     let milliSeconds = time.getMilliseconds();
134     let result = "" + minutes + ":";
135
136     result = result + (seconds < 10 ? "0" : "") + seconds + ".";
137     result = result + (milliSeconds < 10 ? "00" : (milliSeconds < 100 ? "0" : "")) + milliSeconds;
138
139     return result;
140 }
141
142 const fileLoader = (function() {
143     class Loader {
144         constructor() {
145             this.requests = new Map;
146         }
147
148         async _loadInternal(url) {
149             if (!isInBrowser)
150                 return Promise.resolve(readFile(url));
151
152             let fetchResponse = await fetch(new Request(url));
153             if (url.indexOf(".js") !== -1)
154                 return await fetchResponse.text();
155             else if (url.indexOf(".wasm") !== -1)
156                 return await fetchResponse.arrayBuffer();
157
158             throw new Error("should not be reached!");
159         }
160
161         async load(url) {
162             if (this.requests.has(url))
163                 return (await this.requests.get(url));
164
165             let promise = this._loadInternal(url);
166             this.requests.set(url, promise);
167             return (await promise);
168         }
169     }
170     return new Loader;
171 })();
172
173 class Driver {
174     constructor() {
175         this.benchmarks = [];
176     }
177
178     addPlan(plan, BenchmarkClass = DefaultBenchmark) {
179         this.benchmarks.push(new BenchmarkClass(plan));
180     }
181
182     async start() {
183         let statusElement = false;
184         let summaryElement = false;
185         if (isInBrowser) {
186             statusElement = document.getElementById("status");
187             summaryElement = document.getElementById("result-summary");
188             statusElement.innerHTML = `<label>Running...</label>`;
189         } else {
190             console.log("Starting JetStream2");
191         }
192
193         await updateUI();
194
195         let start = Date.now();
196         for (let benchmark of this.benchmarks) {
197             benchmark.updateUIBeforeRun();
198
199             await updateUI();
200
201             try {
202                 await benchmark.run();
203             } catch(e) {
204                 JetStream.reportError(benchmark);
205                 throw e;
206             }
207
208             benchmark.updateUIAfterRun();
209         }
210
211         let totalTime = Date.now() - start;
212         if (measureTotalTimeAsSubtest) {
213             if (isInBrowser)
214                 document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime);
215             else
216                 console.log("Total time:", uiFriendlyNumber(totalTime));
217             allScores.push(totalTime);
218         }
219
220         let allScores = [];
221         for (let benchmark of this.benchmarks)
222             allScores.push(benchmark.score);
223
224         categoryScores = new Map;
225         for (let benchmark of this.benchmarks) {
226             for (let category of Object.keys(benchmark.subTimes()))
227                 categoryScores.set(category, []);
228         }
229
230         for (let benchmark of this.benchmarks) {
231             for (let [category, value] of Object.entries(benchmark.subTimes())) {
232                 let arr = categoryScores.get(category);
233                 arr.push(value);
234             }
235         }
236
237         if (isInBrowser) {
238             summaryElement.classList.add('done');
239             summaryElement.innerHTML = "<div class=\"score\">" + uiFriendlyNumber(geomean(allScores)) + "</div><label>Score</label>";
240             summaryElement.onclick = displayCategoryScores;
241             if (showScoreDetails)
242                 displayCategoryScores();
243             statusElement.innerHTML = '';
244         } else {
245             console.log("\n");
246             for (let [category, scores] of categoryScores)
247                 console.log(`${category}: ${uiFriendlyNumber(geomean(scores))}`);
248
249             console.log("\nTotal Score: ", uiFriendlyNumber(geomean(allScores)), "\n");
250         }
251
252         this.reportScoreToRunBenchmarkRunner();
253     }
254
255     runCode(string)
256     {
257         if (!isInBrowser) {
258             let scripts = string;
259             let globalObject = runString("");
260             globalObject.console = {log:globalObject.print}
261             globalObject.top = {
262                 currentResolve,
263                 currentReject
264             };
265             for (let script of scripts)
266                 globalObject.loadString(script);
267             return globalObject;
268         }
269
270         var magic = document.getElementById("magic");
271         magic.contentDocument.body.textContent = "";
272         magic.contentDocument.body.innerHTML = "<iframe id=\"magicframe\" frameborder=\"0\">";
273
274         var magicFrame = magic.contentDocument.getElementById("magicframe");
275         magicFrame.contentDocument.open();
276         magicFrame.contentDocument.write("<!DOCTYPE html><head><title>benchmark payload</title></head><body>\n" + string + "</body></html>");
277
278         return magicFrame;
279     }
280
281     prepareToRun()
282     {
283         this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
284
285         let text = "";
286         let newBenchmarks = [];
287         for (let benchmark of this.benchmarks) {
288             let id = JSON.stringify(benchmark.constructor.scoreDescription());
289             let description = JSON.parse(id);
290
291             newBenchmarks.push(benchmark);
292             let scoreIds = benchmark.scoreIdentifiers()
293             let overallScoreId = scoreIds.pop();
294
295             if (isInBrowser) {
296                 text +=
297                     `<div class="benchmark" id="benchmark-${benchmark.name}">
298                     <h3 class="benchmark-name"><a href="in-depth.html#${benchmark.name}">${benchmark.name}</a></h3>
299                     <h4 class="score" id="${overallScoreId}">___</h4><p>`;
300                 for (let i = 0; i < scoreIds.length; i++) {
301                     let id = scoreIds[i];
302                     let label = description[i];
303                     text += `<span class="result"><span id="${id}">___</span><label>${label}</label></span>`
304                 }
305                 text += `</p></div>`;
306             }
307         }
308
309         if (!isInBrowser)
310             return;
311
312         for (let f = 0; f < 5; f++)
313             text += `<div class="benchmark fill"></div>`;
314
315         let timestamp = Date.now();
316         document.getElementById('jetstreams').style.backgroundImage = `url('jetstreams.svg?${timestamp}')`;
317         let resultsTable = document.getElementById("results");
318         resultsTable.innerHTML = text;
319
320         document.getElementById("magic").textContent = "";
321         document.addEventListener('keypress', function (e) {
322             if (e.which === 13)
323                 JetStream.start();
324         });
325     }
326
327     reportError(benchmark)
328     {
329         for (let id of benchmark.scoreIdentifiers())
330             document.getElementById(id).innerHTML = "error";
331     }
332
333     async initialize() {
334         await this.fetchResources();
335         this.prepareToRun();
336         if (isInBrowser && window.location.search == '?report=true') {
337             setTimeout(() => this.start(), 4000);
338         }
339     }
340
341     async fetchResources() {
342         for (let benchmark of this.benchmarks)
343             await benchmark.fetchResources();
344
345         if (!isInBrowser)
346             return;
347
348         let statusElement = document.getElementById("status");
349         statusElement.classList.remove('loading');
350         statusElement.innerHTML = `<a href="javascript:JetStream.start()" class="button">Start Test</a>`;
351         statusElement.onclick = () => {
352             statusElement.onclick = null;
353             JetStream.start();
354             return false;
355         }
356     }
357
358     async reportScoreToRunBenchmarkRunner()
359     {
360         if (!isInBrowser)
361             return;
362
363         if (window.location.search !== '?report=true')
364             return;
365
366         let results = {};
367         for (let benchmark of this.benchmarks) {
368             const subResults = {}
369             const subTimes = benchmark.subTimes();
370             for (const name in subTimes) {
371                 subResults[name] = {"metrics": {"Time": {"current": [toTimeValue(subTimes[name])]}}};
372             }
373             results[benchmark.name] = {
374                 "metrics" : {
375                     "Score" : {"current" : [benchmark.score]},
376                     "Time": ["Geometric"],
377                 },
378                 "tests": subResults,
379             };
380         }
381
382         results = {"JetStream2.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
383
384         const content = JSON.stringify(results);
385         await fetch("/report", {
386             method: "POST",
387             heeaders: {
388                 "Content-Type": "application/json",
389                 "Content-Length": content.length,
390                 "Connection": "close",
391             },
392             body: content,
393         });
394     }
395 };
396
397 class Benchmark {
398     constructor(plan)
399     {
400         this.plan = plan;
401         this.iterations = testIterationCount || plan.iterations || defaultIterationCount;
402         this.isAsync = !!plan.isAsync;
403
404         this.scripts = null;
405
406         this._resourcesPromise = null;
407         this.fetchResources();
408     }
409
410     get name() { return this.plan.name; }
411
412     get runnerCode() {
413         return `
414             let __benchmark = new Benchmark(${this.iterations});
415             let results = [];
416             for (let i = 0; i < ${this.iterations}; i++) {
417                 if (__benchmark.prepareForNextIteration)
418                     __benchmark.prepareForNextIteration();
419
420                 let start = Date.now();
421                 __benchmark.runIteration();
422                 let end = Date.now();
423
424                 results.push(Math.max(1, end - start));
425             }
426             if (__benchmark.validate)
427                 __benchmark.validate();
428             top.currentResolve(results);`;
429     }
430
431     processResults() {
432         throw new Error("Subclasses need to implement this");
433     }
434
435     get score() {
436         throw new Error("Subclasses need to implement this");
437     }
438
439     get prerunCode() { return null; }
440
441     async run() {
442         let code;
443         if (isInBrowser)
444             code = "";
445         else
446             code = [];
447
448         let addScript = (text) => {
449             if (isInBrowser)
450                 code += `<script>${text}</script>`;
451             else
452                 code.push(text);
453         };
454
455         let addScriptWithURL = (url) => {
456             if (isInBrowser)
457                 code += `<script src="${url}"></script>`;
458             else
459                 assert(false, "Should not reach here in CLI");
460         };
461
462         addScript(`const isInBrowser = ${isInBrowser}; let performance = {now: Date.now.bind(Date)};`);
463
464         if (!!this.plan.deterministicRandom) {
465             addScript(`
466                 Math.random = (function() {
467                     var seed = 49734321;
468                     return function() {
469                         // Robert Jenkins' 32 bit integer hash function.
470                         seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
471                         seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
472                         seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
473                         seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
474                         seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
475                         seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
476                         return (seed & 0xfffffff) / 0x10000000;
477                     };
478                 })();
479             `);
480
481         }
482
483         if (this.plan.preload) {
484             let str = "";
485             for (let [variableName, blobUrl] of this.preloads)
486                 str += `const ${variableName} = "${blobUrl}";\n`;
487             addScript(str);
488         }
489
490         let prerunCode = this.prerunCode;
491         if (prerunCode)
492             addScript(prerunCode);
493
494         if (preloadResources) {
495             assert(this.scripts && this.scripts.length === this.plan.files.length);
496
497             for (let text of this.scripts)
498                 addScript(text);
499         } else {
500             for (let file of this.plan.files)
501                 addScriptWithURL(file);
502         }
503
504         let promise = new Promise((resolve, reject) => {
505             currentResolve = resolve;
506             currentReject = reject;
507         });
508
509         if (isInBrowser) {
510             code = `
511                 <script> window.onerror = top.currentReject; </script>
512                 ${code}
513             `;
514         }
515         addScript(this.runnerCode);
516
517         this.startTime = new Date();
518
519         if (RAMification)
520             resetMemoryPeak();
521
522         let magicFrame;
523         try {
524             magicFrame = JetStream.runCode(code);
525         } catch(e) {
526             console.log("Error in runCode: ", e);
527             throw e;
528         }
529         let results = await promise;
530
531         this.endTime = new Date();
532
533         if (RAMification) {
534             let memoryFootprint = MemoryFootprint();
535             this.currentFootprint = memoryFootprint.current;
536             this.peakFootprint = memoryFootprint.peak;
537         }
538
539         this.processResults(results);
540         if (isInBrowser)
541             magicFrame.contentDocument.close();
542     }
543
544     fetchResources() {
545         if (this._resourcesPromise)
546             return this._resourcesPromise;
547
548         let filePromises = preloadResources ? this.plan.files.map((file) => fileLoader.load(file)) : [];
549         let preloads = [];
550         let preloadVariableNames = [];
551
552         if (isInBrowser && this.plan.preload) {
553             for (let prop of Object.getOwnPropertyNames(this.plan.preload)) {
554                 preloadVariableNames.push(prop);
555                 preloads.push(this.plan.preload[prop]);
556             }
557         }
558
559         preloads = preloads.map((file) => fileLoader.load(file));
560
561         let p1 = Promise.all(filePromises).then((texts) => {
562             if (!preloadResources)
563                 return;
564             this.scripts = [];
565             assert(texts.length === this.plan.files.length);
566             for (let text of texts)
567                 this.scripts.push(text);
568         });
569
570         let p2 = Promise.all(preloads).then((data) => {
571             this.preloads = [];
572             this.blobs = [];
573             for (let i = 0; i < data.length; ++i) {
574                 let item = data[i];
575
576                 let blob;
577                 if (typeof item === "string") {
578                     blob = new Blob([item], {type : 'application/javascript'});
579                 } else if (item instanceof ArrayBuffer) {
580                     blob = new Blob([item], {type : 'application/octet-stream'});
581                 } else
582                     throw new Error("Unexpected item!");
583
584                 this.blobs.push(blob);
585                 this.preloads.push([preloadVariableNames[i], URL.createObjectURL(blob)]);
586             }
587         });
588
589         this._resourcesPromise = Promise.all([p1, p2]);
590         return this._resourcesPromise;
591     }
592
593     static scoreDescription() { throw new Error("Must be implemented by subclasses."); }
594     scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); }
595
596     updateUIBeforeRun() {
597         if (!isInBrowser) {
598             console.log(`Running ${this.name}:`);
599             return;
600         }
601
602         let containerUI = document.getElementById("results");
603         let resultsBenchmarkUI = document.getElementById(`benchmark-${this.name}`);
604         containerUI.insertBefore(resultsBenchmarkUI, containerUI.firstChild);
605         resultsBenchmarkUI.classList.add("benchmark-running");
606
607         for (let id of this.scoreIdentifiers())
608             document.getElementById(id).innerHTML = "...";
609     }
610
611     updateUIAfterRun() {
612         if (!isInBrowser)
613             return;
614
615         let benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`);
616         benchmarkResultsUI.classList.remove("benchmark-running");
617         benchmarkResultsUI.classList.add("benchmark-done");
618
619     }
620 };
621
622 class DefaultBenchmark extends Benchmark {
623     constructor(...args) {
624         super(...args);
625
626         this.worstCaseCount = this.plan.worstCaseCount || defaultWorstCaseCount;
627         this.firstIteration = null;
628         this.worst4 = null;
629         this.average = null;
630     }
631
632     processResults(results) {
633         function copyArray(a) {
634             let result = [];
635             for (let x of a)
636                 result.push(x);
637             return result;
638         }
639         results = copyArray(results);
640
641         this.firstIteration = toScore(results[0]);
642
643         results = results.slice(1);
644         results.sort((a, b) => a < b ? 1 : -1);
645         for (let i = 0; i + 1 < results.length; ++i)
646             assert(results[i] >= results[i + 1]);
647
648         let worstCase = [];
649         for (let i = 0; i < this.worstCaseCount; ++i)
650             worstCase.push(results[i]);
651         this.worst4 = toScore(mean(worstCase));
652         this.average = toScore(mean(results));
653     }
654
655     get score() {
656         return geomean([this.firstIteration, this.worst4, this.average]);
657     }
658
659     subTimes() {
660         return {
661             "First": this.firstIteration,
662             "Worst": this.worst4,
663             "Average": this.average,
664         };
665     }
666
667     static scoreDescription() {
668         return ["First", "Worst", "Average", "Score"];
669     }
670
671     scoreIdentifiers() {
672         return [firstID(this), worst4ID(this), avgID(this), scoreID(this)];
673     }
674
675     updateUIAfterRun() {
676         super.updateUIAfterRun();
677
678         if (isInBrowser) {
679             document.getElementById(firstID(this)).innerHTML = uiFriendlyNumber(this.firstIteration);
680             document.getElementById(worst4ID(this)).innerHTML = uiFriendlyNumber(this.worst4);
681             document.getElementById(avgID(this)).innerHTML = uiFriendlyNumber(this.average);
682             document.getElementById(scoreID(this)).innerHTML = uiFriendlyNumber(this.score);
683             return;
684         }
685
686         print("    Startup:", uiFriendlyNumber(this.firstIteration));
687         print("    Worst Case:", uiFriendlyNumber(this.worst4));
688         print("    Average:", uiFriendlyNumber(this.average));
689         print("    Score:", uiFriendlyNumber(this.score));
690         if (RAMification) {
691             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
692             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
693         }
694         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
695     }
696 }
697
698 class AsyncBenchmark extends DefaultBenchmark {
699     get runnerCode() {
700         return `
701         async function doRun() {
702             let __benchmark = new Benchmark();
703             let results = [];
704             for (let i = 0; i < ${this.iterations}; i++) {
705                 let start = Date.now();
706                 await __benchmark.runIteration();
707                 let end = Date.now();
708                 results.push(Math.max(1, end - start));
709             }
710             if (__benchmark.validate)
711                 __benchmark.validate();
712             top.currentResolve(results);
713         }
714         doRun();`
715     }
716 };
717
718 class WSLBenchmark extends Benchmark {
719     constructor(...args) {
720         super(...args);
721
722         this.stdlib = null;
723         this.mainRun = null;
724     }
725
726     processResults(results) {
727         this.stdlib = toScore(results[0]);
728         this.mainRun = toScore(results[1]);
729     }
730
731     get score() {
732         return geomean([this.stdlib, this.mainRun]);
733     }
734
735     get runnerCode() {
736         return `
737             let benchmark = new Benchmark();
738             let results = [];
739             {
740                 let start = Date.now();
741                 benchmark.buildStdlib();
742                 results.push(Date.now() - start);
743             }
744
745             {
746                 let start = Date.now();
747                 benchmark.run();
748                 results.push(Date.now() - start);
749             }
750
751             top.currentResolve(results);
752             `;
753     }
754
755     subTimes() {
756         return {
757             "Stdlib": this.stdlib,
758             "MainRun": this.mainRun,
759         };
760     }
761
762     static scoreDescription() {
763         return ["Stdlib", "MainRun", "Score"];
764     }
765
766     scoreIdentifiers() {
767         return ["wsl-stdlib-score", "wsl-tests-score", "wsl-score-score"];
768     }
769
770     updateUIAfterRun() {
771         super.updateUIAfterRun();
772
773         if (isInBrowser) {
774             document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyNumber(this.stdlib);
775             document.getElementById("wsl-tests-score").innerHTML = uiFriendlyNumber(this.mainRun);
776             document.getElementById("wsl-score-score").innerHTML = uiFriendlyNumber(this.score);
777             return;
778         }
779
780         print("    Stdlib:", uiFriendlyNumber(this.stdlib));
781         print("    Tests:", uiFriendlyNumber(this.mainRun));
782         print("    Score:", uiFriendlyNumber(this.score));
783         if (RAMification) {
784             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
785             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
786         }
787         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
788     }
789 };
790
791 class WasmBenchmark extends Benchmark {
792     constructor(...args) {
793         super(...args);
794
795         this.startupTime = null;
796         this.runTime = null;
797     }
798
799     processResults(results) {
800         this.startupTime = toScore(results[0]);
801         this.runTime = toScore(results[1]);
802     }
803
804     get score() {
805         return geomean([this.startupTime, this.runTime]);
806     }
807
808     get wasmPath() {
809         return this.plan.wasmPath;
810     }
811
812     get prerunCode() {
813         let str = `
814             let verbose = false;
815
816             let compileTime = null;
817             let runTime = null;
818
819             let globalObject = this;
820
821             globalObject.benchmarkTime = Date.now.bind(Date);
822
823             globalObject.reportCompileTime = (t) => {
824                 if (compileTime !== null)
825                     throw new Error("called report compile time twice");
826                 compileTime = t;
827             };
828
829             globalObject.reportRunTime = (t) => {
830                 if (runTime !== null)
831                     throw new Error("called report run time twice")
832                 runTime = t;
833                 top.currentResolve([compileTime, runTime]);
834             };
835
836             abort = quit = function() {
837                 if (verbose)
838                     console.log('Intercepted quit/abort');
839             };
840
841             oldPrint = globalObject.print;
842             globalObject.print = globalObject.printErr = (...args) => {
843                 if (verbose)
844                     console.log('Intercepted print: ', ...args);
845             };
846
847             let Module = {
848                 preRun: [],
849                 postRun: [],
850                 print: function() { },
851                 printErr: function() { },
852                 setStatus: function(text) {
853                 },
854                 totalDependencies: 0,
855                 monitorRunDependencies: function(left) {
856                     this.totalDependencies = Math.max(this.totalDependencies, left);
857                     Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
858                 }
859             };
860             globalObject.Module = Module;
861             `;
862         return str;
863     }
864
865     get runnerCode() {
866         let str = "";
867         if (isInBrowser) {
868             str += `
869                 var xhr = new XMLHttpRequest();
870                 xhr.open('GET', wasmBlobURL, true);
871                 xhr.responseType = 'arraybuffer';
872                 xhr.onload = function() {
873                     Module.wasmBinary = xhr.response;
874                     doRun();
875                 };
876                 xhr.send(null);
877             `;
878         } else {
879             str += `
880             Module.wasmBinary = read("${this.wasmPath}", "binary");
881             globalObject.read = (...args) => {
882                 console.log("should not be inside read: ", ...args);
883                 throw new Error;
884             };
885
886             Module.setStatus = null;
887             Module.monitorRunDependencies = null;
888
889             Promise.resolve(42).then(() => {
890                 try {
891                     doRun();
892                 } catch(e) {
893                     console.log("error running wasm:", e);
894                     throw e;
895                 }
896             })
897             `;
898         }
899         return str;
900     }
901
902     subTimes() {
903         return {
904             "Startup": this.startupTime,
905             "Runtime": this.runTime,
906         };
907     }
908
909     static scoreDescription() {
910         return ["Startup", "Runtime", "Score"];
911     }
912
913     get startupID() {
914         return `wasm-startup-id${this.name}`;
915     }
916     get runID() {
917         return `wasm-run-id${this.name}`;
918     }
919     get scoreID() {
920         return `wasm-score-id${this.name}`;
921     }
922
923     scoreIdentifiers() {
924         return [this.startupID, this.runID, this.scoreID];
925     }
926
927     updateUIAfterRun() {
928         super.updateUIAfterRun();
929
930         if (isInBrowser) {
931             document.getElementById(this.startupID).innerHTML = uiFriendlyNumber(this.startupTime);
932             document.getElementById(this.runID).innerHTML = uiFriendlyNumber(this.runTime);
933             document.getElementById(this.scoreID).innerHTML = uiFriendlyNumber(this.score);
934             return;
935         }
936         print("    Startup:", uiFriendlyNumber(this.startupTime));
937         print("    Run time:", uiFriendlyNumber(this.runTime));
938         if (RAMification) {
939             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
940             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
941         }
942         print("    Score:", uiFriendlyNumber(this.score));
943     }
944 };
945
946 const ARESGroup = Symbol.for("ARES");
947 const CDJSGroup = Symbol.for("CDJS");
948 const CodeLoadGroup = Symbol.for("CodeLoad");
949 const LuaJSFightGroup = Symbol.for("LuaJSFight");
950 const OctaneGroup = Symbol.for("Octane");
951 const RexBenchGroup = Symbol.for("RexBench");
952 const SeaMonsterGroup = Symbol.for("SeaMonster");
953 const SimpleGroup = Symbol.for("Simple");
954 const SunSpiderGroup = Symbol.for("SunSpider");
955 const WasmGroup = Symbol.for("Wasm");
956 const WorkerTestsGroup = Symbol.for("WorkerTests");
957 const WSLGroup = Symbol.for("WSL");
958 const WTBGroup = Symbol.for("WTB");
959
960
961 let testPlans = [
962     // ARES
963     {
964         name: "Air",
965         files: [
966             "./ARES-6/Air/symbols.js"
967             , "./ARES-6/Air/tmp_base.js"
968             , "./ARES-6/Air/arg.js"
969             , "./ARES-6/Air/basic_block.js"
970             , "./ARES-6/Air/code.js"
971             , "./ARES-6/Air/frequented_block.js"
972             , "./ARES-6/Air/inst.js"
973             , "./ARES-6/Air/opcode.js"
974             , "./ARES-6/Air/reg.js"
975             , "./ARES-6/Air/stack_slot.js"
976             , "./ARES-6/Air/tmp.js"
977             , "./ARES-6/Air/util.js"
978             , "./ARES-6/Air/custom.js"
979             , "./ARES-6/Air/liveness.js"
980             , "./ARES-6/Air/insertion_set.js"
981             , "./ARES-6/Air/allocate_stack.js"
982             , "./ARES-6/Air/payload-gbemu-executeIteration.js"
983             , "./ARES-6/Air/payload-imaging-gaussian-blur-gaussianBlur.js"
984             , "./ARES-6/Air/payload-airjs-ACLj8C.js"
985             , "./ARES-6/Air/payload-typescript-scanIdentifier.js"
986             , "./ARES-6/Air/benchmark.js"
987         ],
988         testGroup: ARESGroup
989     },
990     {
991         name: "Basic",
992         files: [
993             "./ARES-6/Basic/ast.js"
994             , "./ARES-6/Basic/basic.js"
995             , "./ARES-6/Basic/caseless_map.js"
996             , "./ARES-6/Basic/lexer.js"
997             , "./ARES-6/Basic/number.js"
998             , "./ARES-6/Basic/parser.js"
999             , "./ARES-6/Basic/random.js"
1000             , "./ARES-6/Basic/state.js"
1001             , "./ARES-6/Basic/util.js"
1002             , "./ARES-6/Basic/benchmark.js"
1003         ],
1004         testGroup: ARESGroup
1005     },
1006     {
1007         name: "ML",
1008         files: [
1009             "./ARES-6/ml/index.js"
1010             , "./ARES-6/ml/benchmark.js"
1011         ],
1012         iterations: 60,
1013         testGroup: ARESGroup
1014     },
1015     {
1016         name: "Babylon",
1017         files: [
1018             "./ARES-6/Babylon/index.js"
1019             , "./ARES-6/Babylon/benchmark.js"
1020         ],
1021         preload: {
1022             airBlob: "./ARES-6/Babylon/air-blob.js",
1023             basicBlob: "./ARES-6/Babylon/basic-blob.js",
1024             inspectorBlob: "./ARES-6/Babylon/inspector-blob.js",
1025             babylonBlob: "./ARES-6/Babylon/babylon-blob.js"
1026         },
1027         testGroup: ARESGroup
1028     },
1029     // CDJS
1030     {
1031         name: "cdjs",
1032         files: [
1033             "./cdjs/constants.js"
1034             , "./cdjs/util.js"
1035             , "./cdjs/red_black_tree.js"
1036             , "./cdjs/call_sign.js"
1037             , "./cdjs/vector_2d.js"
1038             , "./cdjs/vector_3d.js"
1039             , "./cdjs/motion.js"
1040             , "./cdjs/reduce_collision_set.js"
1041             , "./cdjs/simulator.js"
1042             , "./cdjs/collision.js"
1043             , "./cdjs/collision_detector.js"
1044             , "./cdjs/benchmark.js"
1045         ],
1046         iterations: 60,
1047         worstCaseCount: 3,
1048         testGroup: CDJSGroup
1049     },
1050     // CodeLoad
1051     {
1052         name: "first-inspector-code-load",
1053         files: [
1054             "./code-load/code-first-load.js"
1055         ],
1056         preload: {
1057             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1058         },
1059         testGroup: CodeLoadGroup
1060     },
1061     {
1062         name: "multi-inspector-code-load",
1063         files: [
1064             "./code-load/code-multi-load.js"
1065         ],
1066         preload: {
1067             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1068         },
1069         testGroup: CodeLoadGroup
1070     },
1071     // Octane
1072     {
1073         name: "Box2D",
1074         files: [
1075             "./Octane/box2d.js"
1076         ],
1077         deterministicRandom: true,
1078         testGroup: OctaneGroup
1079     },
1080     {
1081         name: "octane-code-load",
1082         files: [
1083             "./Octane/code-first-load.js"
1084         ],
1085         deterministicRandom: true,
1086         testGroup: OctaneGroup
1087     },
1088     {
1089         name: "crypto",
1090         files: [
1091             "./Octane/crypto.js"
1092         ],
1093         deterministicRandom: true,
1094         testGroup: OctaneGroup
1095     },
1096     {
1097         name: "delta-blue",
1098         files: [
1099             "./Octane/deltablue.js"
1100         ],
1101         deterministicRandom: true,
1102         testGroup: OctaneGroup
1103     },
1104     {
1105         name: "earley-boyer",
1106         files: [
1107             "./Octane/earley-boyer.js"
1108         ],
1109         deterministicRandom: true,
1110         testGroup: OctaneGroup
1111     },
1112     {
1113         name: "gbemu",
1114         files: [
1115             "./Octane/gbemu-part1.js"
1116             , "./Octane/gbemu-part2.js"
1117         ],
1118         deterministicRandom: true,
1119         testGroup: OctaneGroup
1120     },
1121     {
1122         name: "mandreel",
1123         files: [
1124             "./Octane/mandreel.js"
1125         ],
1126         iterations: 80,
1127         deterministicRandom: true,
1128         testGroup: OctaneGroup
1129     },
1130     {
1131         name: "navier-stokes",
1132         files: [
1133             "./Octane/navier-stokes.js"
1134         ],
1135         deterministicRandom: true,
1136         testGroup: OctaneGroup
1137     },
1138     {
1139         name: "pdfjs",
1140         files: [
1141             "./Octane/pdfjs.js"
1142         ],
1143         deterministicRandom: true,
1144         testGroup: OctaneGroup
1145     },
1146     {
1147         name: "raytrace",
1148         files: [
1149             "./Octane/raytrace.js"
1150         ],
1151         deterministicRandom: true,
1152         testGroup: OctaneGroup
1153     },
1154     {
1155         name: "regexp",
1156         files: [
1157             "./Octane/regexp.js"
1158         ],
1159         deterministicRandom: true,
1160         testGroup: OctaneGroup
1161     },
1162     {
1163         name: "richards",
1164         files: [
1165             "./Octane/richards.js"
1166         ],
1167         deterministicRandom: true,
1168         testGroup: OctaneGroup
1169     },
1170     {
1171         name: "splay",
1172         files: [
1173             "./Octane/splay.js"
1174         ],
1175         deterministicRandom: true,
1176         testGroup: OctaneGroup
1177     },
1178     {
1179         name: "typescript",
1180         files: [
1181             "./Octane/typescript-compiler.js"
1182             , "./Octane/typescript-input.js"
1183             , "./Octane/typescript.js"
1184         ],
1185         iterations: 15,
1186         worstCaseCount: 2,
1187         deterministicRandom: true,
1188         testGroup: OctaneGroup
1189     },
1190     {
1191         name: "octane-zlib",
1192         files: [
1193             "./Octane/zlib-data.js"
1194             , "./Octane/zlib.js"
1195         ],
1196         iterations: 15,
1197         worstCaseCount: 2,
1198         deterministicRandom: true,
1199         testGroup: OctaneGroup
1200     },
1201     // RexBench
1202     {
1203         name: "FlightPlanner",
1204         files: [
1205             "./RexBench/FlightPlanner/airways.js"
1206             , "./RexBench/FlightPlanner/waypoints.js"
1207             , "./RexBench/FlightPlanner/flight_planner.js"
1208             , "./RexBench/FlightPlanner/expectations.js"
1209             , "./RexBench/FlightPlanner/benchmark.js"
1210         ],
1211         testGroup: RexBenchGroup
1212     },
1213     {
1214         name: "OfflineAssembler",
1215         files: [
1216             "./RexBench/OfflineAssembler/registers.js"
1217             , "./RexBench/OfflineAssembler/instructions.js"
1218             , "./RexBench/OfflineAssembler/ast.js"
1219             , "./RexBench/OfflineAssembler/parser.js"
1220             , "./RexBench/OfflineAssembler/file.js"
1221             , "./RexBench/OfflineAssembler/LowLevelInterpreter.js"
1222             , "./RexBench/OfflineAssembler/LowLevelInterpreter32_64.js"
1223             , "./RexBench/OfflineAssembler/LowLevelInterpreter64.js"
1224             , "./RexBench/OfflineAssembler/InitBytecodes.js"
1225             , "./RexBench/OfflineAssembler/expected.js"
1226             , "./RexBench/OfflineAssembler/benchmark.js"
1227         ],
1228         iterations: 80,
1229         testGroup: RexBenchGroup
1230     },
1231     {
1232         name: "UniPoker",
1233         files: [
1234             "./RexBench/UniPoker/poker.js"
1235             , "./RexBench/UniPoker/expected.js"
1236             , "./RexBench/UniPoker/benchmark.js"
1237         ],
1238         deterministicRandom: true,
1239         testGroup: RexBenchGroup
1240     },
1241     // Simple
1242     {
1243         name: "async-fs",
1244         files: [
1245             "./simple/file-system.js"
1246         ],
1247         iterations: 40,
1248         worstCaseCount: 3,
1249         benchmarkClass: AsyncBenchmark,
1250         testGroup: SimpleGroup
1251     },
1252     {
1253         name: "float-mm.c",
1254         files: [
1255             "./simple/float-mm.c.js"
1256         ],
1257         iterations: 15,
1258         worstCaseCount: 2,
1259         testGroup: SimpleGroup
1260     },
1261     {
1262         name: "hash-map",
1263         files: [
1264             "./simple/hash-map.js"
1265         ],
1266         testGroup: SimpleGroup
1267     },
1268     // SeaMonster
1269     {
1270         name: "ai-astar",
1271         files: [
1272             "./SeaMonster/ai-astar.js"
1273         ],
1274         testGroup: SeaMonsterGroup
1275     },
1276     {
1277         name: "gaussian-blur",
1278         files: [
1279             "./SeaMonster/gaussian-blur.js"
1280         ],
1281         testGroup: SeaMonsterGroup
1282     },
1283     {
1284         name: "stanford-crypto-aes",
1285         files: [
1286             "./SeaMonster/sjlc.js"
1287             , "./SeaMonster/stanford-crypto-aes.js"
1288         ],
1289         testGroup: SeaMonsterGroup
1290     },
1291     {
1292         name: "stanford-crypto-pbkdf2",
1293         files: [
1294             "./SeaMonster/sjlc.js"
1295             , "./SeaMonster/stanford-crypto-pbkdf2.js"
1296         ],
1297         testGroup: SeaMonsterGroup
1298     },
1299     {
1300         name: "stanford-crypto-sha256",
1301         files: [
1302             "./SeaMonster/sjlc.js"
1303             , "./SeaMonster/stanford-crypto-sha256.js"
1304         ],
1305         testGroup: SeaMonsterGroup
1306     },
1307     {
1308         name: "json-stringify-inspector",
1309         files: [
1310             "./SeaMonster/inspector-json-payload.js"
1311             , "./SeaMonster/json-stringify-inspector.js"
1312         ],
1313         iterations: 20,
1314         worstCaseCount: 2,
1315         testGroup: SeaMonsterGroup
1316     },
1317     {
1318         name: "json-parse-inspector",
1319         files: [
1320             "./SeaMonster/inspector-json-payload.js"
1321             , "./SeaMonster/json-parse-inspector.js"
1322         ],
1323         iterations: 20,
1324         worstCaseCount: 2,
1325         testGroup: SeaMonsterGroup
1326     },
1327     // Wasm
1328     {
1329         name: "HashSet-wasm",
1330         wasmPath: "./wasm/HashSet.wasm",
1331         files: [
1332             "./wasm/HashSet.js"
1333         ],
1334         preload: {
1335             wasmBlobURL: "./wasm/HashSet.wasm"
1336         },
1337         benchmarkClass: WasmBenchmark,
1338         testGroup: WasmGroup
1339     },
1340     {
1341         name: "tsf-wasm",
1342         wasmPath: "./wasm/tsf.wasm",
1343         files: [
1344             "./wasm/tsf.js"
1345         ],
1346         preload: {
1347             wasmBlobURL: "./wasm/tsf.wasm"
1348         },
1349         benchmarkClass: WasmBenchmark,
1350         testGroup: WasmGroup
1351     },
1352     {
1353         name: "quicksort-wasm",
1354         wasmPath: "./wasm/quicksort.wasm",
1355         files: [
1356             "./wasm/quicksort.js"
1357         ],
1358         preload: {
1359             wasmBlobURL: "./wasm/quicksort.wasm"
1360         },
1361         benchmarkClass: WasmBenchmark,
1362         testGroup: WasmGroup
1363     },
1364     {
1365         name: "gcc-loops-wasm",
1366         wasmPath: "./wasm/gcc-loops.wasm",
1367         files: [
1368             "./wasm/gcc-loops.js"
1369         ],
1370         preload: {
1371             wasmBlobURL: "./wasm/gcc-loops.wasm"
1372         },
1373         benchmarkClass: WasmBenchmark,
1374         testGroup: WasmGroup
1375     },
1376     {
1377         name: "richards-wasm",
1378         wasmPath: "./wasm/richards.wasm",
1379         files: [
1380             "./wasm/richards.js"
1381         ],
1382         preload: {
1383             wasmBlobURL: "./wasm/richards.wasm"
1384         },
1385         benchmarkClass: WasmBenchmark,
1386         testGroup: WasmGroup
1387     },
1388     // WorkerTests
1389     {
1390         name: "bomb-workers",
1391         files: [
1392             "./worker/bomb.js"
1393         ],
1394         iterations: 80,
1395         preload: {
1396             rayTrace3D: "./worker/bomb-subtests/3d-raytrace.js"
1397             , accessNbody: "./worker/bomb-subtests/access-nbody.js"
1398             , morph3D: "./worker/bomb-subtests/3d-morph.js"
1399             , cube3D: "./worker/bomb-subtests/3d-cube.js"
1400             , accessFunnkuch: "./worker/bomb-subtests/access-fannkuch.js"
1401             , accessBinaryTrees: "./worker/bomb-subtests/access-binary-trees.js"
1402             , accessNsieve: "./worker/bomb-subtests/access-nsieve.js"
1403             , bitopsBitwiseAnd: "./worker/bomb-subtests/bitops-bitwise-and.js"
1404             , bitopsNsieveBits: "./worker/bomb-subtests/bitops-nsieve-bits.js"
1405             , controlflowRecursive: "./worker/bomb-subtests/controlflow-recursive.js"
1406             , bitops3BitBitsInByte: "./worker/bomb-subtests/bitops-3bit-bits-in-byte.js"
1407             , botopsBitsInByte: "./worker/bomb-subtests/bitops-bits-in-byte.js"
1408             , cryptoAES: "./worker/bomb-subtests/crypto-aes.js"
1409             , cryptoMD5: "./worker/bomb-subtests/crypto-md5.js"
1410             , cryptoSHA1: "./worker/bomb-subtests/crypto-sha1.js"
1411             , dateFormatTofte: "./worker/bomb-subtests/date-format-tofte.js"
1412             , dateFormatXparb: "./worker/bomb-subtests/date-format-xparb.js"
1413             , mathCordic: "./worker/bomb-subtests/math-cordic.js"
1414             , mathPartialSums: "./worker/bomb-subtests/math-partial-sums.js"
1415             , mathSpectralNorm: "./worker/bomb-subtests/math-spectral-norm.js"
1416             , stringBase64: "./worker/bomb-subtests/string-base64.js"
1417             , stringFasta: "./worker/bomb-subtests/string-fasta.js"
1418             , stringValidateInput: "./worker/bomb-subtests/string-validate-input.js"
1419             , stringTagcloud: "./worker/bomb-subtests/string-tagcloud.js"
1420             , stringUnpackCode: "./worker/bomb-subtests/string-unpack-code.js"
1421             , regexpDNA: "./worker/bomb-subtests/regexp-dna.js"
1422         },
1423         benchmarkClass: AsyncBenchmark,
1424         testGroup: WorkerTestsGroup
1425     },
1426     {
1427         name: "segmentation",
1428         files: [
1429             "./worker/segmentation.js"
1430         ],
1431         preload: {
1432             asyncTaskBlob: "./worker/async-task.js"
1433         },
1434         iterations: 36,
1435         worstCaseCount: 3,
1436         benchmarkClass: AsyncBenchmark,
1437         testGroup: WorkerTestsGroup
1438     },
1439     // WSL
1440     {
1441         name: "WSL",
1442         files: ["./WSL/Node.js" ,"./WSL/Type.js" ,"./WSL/ReferenceType.js" ,"./WSL/Value.js" ,"./WSL/Expression.js" ,"./WSL/Rewriter.js" ,"./WSL/Visitor.js" ,"./WSL/CreateLiteral.js" ,"./WSL/CreateLiteralType.js" ,"./WSL/PropertyAccessExpression.js" ,"./WSL/AddressSpace.js" ,"./WSL/AnonymousVariable.js" ,"./WSL/ArrayRefType.js" ,"./WSL/ArrayType.js" ,"./WSL/Assignment.js" ,"./WSL/AutoWrapper.js" ,"./WSL/Block.js" ,"./WSL/BoolLiteral.js" ,"./WSL/Break.js" ,"./WSL/CallExpression.js" ,"./WSL/CallFunction.js" ,"./WSL/Check.js" ,"./WSL/CheckLiteralTypes.js" ,"./WSL/CheckLoops.js" ,"./WSL/CheckRecursiveTypes.js" ,"./WSL/CheckRecursion.js" ,"./WSL/CheckReturns.js" ,"./WSL/CheckUnreachableCode.js" ,"./WSL/CheckWrapped.js" ,"./WSL/Checker.js" ,"./WSL/CloneProgram.js" ,"./WSL/CommaExpression.js" ,"./WSL/ConstexprFolder.js" ,"./WSL/ConstexprTypeParameter.js" ,"./WSL/Continue.js" ,"./WSL/ConvertPtrToArrayRefExpression.js" ,"./WSL/DereferenceExpression.js" ,"./WSL/DoWhileLoop.js" ,"./WSL/DotExpression.js" ,"./WSL/DoubleLiteral.js" ,"./WSL/DoubleLiteralType.js" ,"./WSL/EArrayRef.js" ,"./WSL/EBuffer.js" ,"./WSL/EBufferBuilder.js" ,"./WSL/EPtr.js" ,"./WSL/EnumLiteral.js" ,"./WSL/EnumMember.js" ,"./WSL/EnumType.js" ,"./WSL/EvaluationCommon.js" ,"./WSL/Evaluator.js" ,"./WSL/ExpressionFinder.js" ,"./WSL/ExternalOrigin.js" ,"./WSL/Field.js" ,"./WSL/FindHighZombies.js" ,"./WSL/FlattenProtocolExtends.js" ,"./WSL/FlattenedStructOffsetGatherer.js" ,"./WSL/FloatLiteral.js" ,"./WSL/FloatLiteralType.js" ,"./WSL/FoldConstexprs.js" ,"./WSL/ForLoop.js" ,"./WSL/Func.js" ,"./WSL/FuncDef.js" ,"./WSL/FuncInstantiator.js" ,"./WSL/FuncParameter.js" ,"./WSL/FunctionLikeBlock.js" ,"./WSL/HighZombieFinder.js" ,"./WSL/IdentityExpression.js" ,"./WSL/IfStatement.js" ,"./WSL/IndexExpression.js" ,"./WSL/InferTypesForCall.js" ,"./WSL/Inline.js" ,"./WSL/Inliner.js" ,"./WSL/InstantiateImmediates.js" ,"./WSL/IntLiteral.js" ,"./WSL/IntLiteralType.js" ,"./WSL/Intrinsics.js" ,"./WSL/LateChecker.js" ,"./WSL/Lexer.js" ,"./WSL/LexerToken.js" ,"./WSL/LiteralTypeChecker.js" ,"./WSL/LogicalExpression.js" ,"./WSL/LogicalNot.js" ,"./WSL/LoopChecker.js" ,"./WSL/MakeArrayRefExpression.js" ,"./WSL/MakePtrExpression.js" ,"./WSL/NameContext.js" ,"./WSL/NameFinder.js" ,"./WSL/NameResolver.js" ,"./WSL/NativeFunc.js" ,"./WSL/NativeFuncInstance.js" ,"./WSL/NativeType.js" ,"./WSL/NativeTypeInstance.js" ,"./WSL/NormalUsePropertyResolver.js" ,"./WSL/NullLiteral.js" ,"./WSL/NullType.js" ,"./WSL/OriginKind.js" ,"./WSL/OverloadResolutionFailure.js" ,"./WSL/Parse.js" ,"./WSL/Prepare.js" ,"./WSL/Program.js" ,"./WSL/ProgramWithUnnecessaryThingsRemoved.js" ,"./WSL/PropertyResolver.js" ,"./WSL/Protocol.js" ,"./WSL/ProtocolDecl.js" ,"./WSL/ProtocolFuncDecl.js" ,"./WSL/ProtocolRef.js" ,"./WSL/PtrType.js" ,"./WSL/ReadModifyWriteExpression.js" ,"./WSL/RecursionChecker.js" ,"./WSL/RecursiveTypeChecker.js" ,"./WSL/ResolveNames.js" ,"./WSL/ResolveOverloadImpl.js" ,"./WSL/ResolveProperties.js" ,"./WSL/ResolveTypeDefs.js" ,"./WSL/Return.js" ,"./WSL/ReturnChecker.js" ,"./WSL/ReturnException.js" ,"./WSL/StandardLibrary.js" ,"./WSL/StatementCloner.js" ,"./WSL/StructLayoutBuilder.js" ,"./WSL/StructType.js" ,"./WSL/Substitution.js" ,"./WSL/SwitchCase.js" ,"./WSL/SwitchStatement.js" ,"./WSL/SynthesizeEnumFunctions.js" ,"./WSL/SynthesizeStructAccessors.js" ,"./WSL/TrapStatement.js" ,"./WSL/TypeDef.js" ,"./WSL/TypeDefResolver.js" ,"./WSL/TypeOrVariableRef.js" ,"./WSL/TypeParameterRewriter.js" ,"./WSL/TypeRef.js" ,"./WSL/TypeVariable.js" ,"./WSL/TypeVariableTracker.js" ,"./WSL/TypedValue.js" ,"./WSL/UintLiteral.js" ,"./WSL/UintLiteralType.js" ,"./WSL/UnificationContext.js" ,"./WSL/UnreachableCodeChecker.js" ,"./WSL/VariableDecl.js" ,"./WSL/VariableRef.js" ,"./WSL/VisitingSet.js" ,"./WSL/WSyntaxError.js" ,"./WSL/WTrapError.js" ,"./WSL/WTypeError.js" ,"./WSL/WhileLoop.js" ,"./WSL/WrapChecker.js", "./WSL/Test.js"],
1443         benchmarkClass: WSLBenchmark,
1444         testGroup: WSLGroup
1445     }
1446 ];
1447
1448 // LuaJSFight tests
1449 let luaJSFightTests = [
1450     "hello_world"
1451     , "list_search"
1452     , "lists"
1453     , "string_lists"
1454 ];
1455 for (let test of luaJSFightTests) {
1456     testPlans.push({
1457         name: `${test}-LJF`,
1458         files: [
1459             `./LuaJSFight/${test}.js`
1460         ],
1461         testGroup: LuaJSFightGroup
1462     });
1463 }
1464
1465 // SunSpider tests
1466 let sunSpiderTests = [
1467     "3d-cube"
1468     , "3d-raytrace"
1469     , "base64"
1470     , "crypto-aes"
1471     , "crypto-md5"
1472     , "crypto-sha1"
1473     , "date-format-tofte"
1474     , "date-format-xparb"
1475     , "n-body"
1476     , "regex-dna"
1477     , "string-unpack-code"
1478     , "tagcloud"
1479 ];
1480 for (let test of sunSpiderTests) {
1481     testPlans.push({
1482         name: `${test}-SP`,
1483         files: [
1484             `./SunSpider/${test}.js`
1485         ],
1486         testGroup: SunSpiderGroup
1487     });
1488 }
1489
1490 // WTB (Web Tooling Benchmark) tests
1491 let WTBTests = [
1492     "acorn"
1493     , "babylon"
1494     , "chai"
1495     , "coffeescript"
1496     , "espree"
1497     , "jshint"
1498     , "lebab"
1499     , "prepack"
1500     , "uglify-js"
1501 ];
1502 for (let name of WTBTests) {
1503     testPlans.push({
1504         name: `${name}-wtb`,
1505         files: [
1506             isInBrowser ? "./web-tooling-benchmark/browser.js" : "./web-tooling-benchmark/cli.js"
1507             , `./web-tooling-benchmark/${name}.js`
1508         ],
1509         iterations: 5,
1510         worstCaseCount: 1,
1511         testGroup: WTBGroup
1512     });
1513 }
1514
1515
1516 let testsByName = new Map();
1517 let testsByGroup = new Map();
1518
1519 for (let plan of testPlans) {
1520     let testName = plan.name;
1521
1522     if (testsByName.has(plan.name))
1523         throw "Duplicate test plan with name \"" + testName + "\"";
1524     else
1525         testsByName.set(testName, plan);
1526
1527     let group = plan.testGroup;
1528
1529     if (testsByGroup.has(group))
1530         testsByGroup.get(group).push(testName);
1531     else
1532         testsByGroup.set(group, [testName]);
1533 }
1534
1535 this.JetStream = new Driver();
1536
1537 function addTestByName(testName)
1538 {
1539     let plan = testsByName.get(testName);
1540
1541     if (plan)
1542         JetStream.addPlan(plan, plan.benchmarkClass);
1543     else
1544         throw "Couldn't find test named \"" +  testName + "\"";
1545 }
1546
1547 function addTestsByGroup(group)
1548 {
1549     let testList = testsByGroup.get(group);
1550
1551     if (!testList)
1552         throw "Couldn't find test group named: \"" + Symbol.keyFor(group) + "\"";
1553
1554     for (let testName of testList)
1555         addTestByName(testName);
1556 }
1557
1558 function processTestList(testList)
1559 {
1560     let tests = [];
1561
1562     if (testList instanceof Array)
1563         tests = testList;
1564     else
1565         tests = testList.split(/[\s,]/);
1566
1567     for (let testName of tests) {
1568         let groupTest = testsByGroup.get(Symbol.for(testName));
1569
1570         if (groupTest) {
1571             for (let testName of groupTest)
1572                 addTestByName(testName);
1573         } else
1574             addTestByName(testName);
1575     }
1576 }
1577
1578 let runOctane = true;
1579 let runARES = true;
1580 let runWSL = true;
1581 let runRexBench = true;
1582 let runWTB = true;
1583 let runSunSpider = true;
1584 let runSimple = true;
1585 let runCDJS = true;
1586 let runWorkerTests = !!isInBrowser;
1587 let runSeaMonster = true;
1588 let runCodeLoad = true;
1589 let runWasm = true;
1590
1591 if (false) {
1592     runOctane = false;
1593     runARES = false;
1594     runWSL = false;
1595     runRexBench = false;
1596     runWTB = false;
1597     runSunSpider = false;
1598     runSimple = false;
1599     runCDJS = false;
1600     runWorkerTests = false;
1601     runSeaMonster = false;
1602     runCodeLoad = false;
1603     runWasm = false;
1604 }
1605
1606 if (typeof testList !== "undefined") {
1607     processTestList(testList);
1608 } else {
1609     if (runARES)
1610         addTestsByGroup(ARESGroup);
1611
1612     if (runCDJS)
1613         addTestsByGroup(CDJSGroup);
1614
1615     if (runCodeLoad)
1616         addTestsByGroup(CodeLoadGroup);
1617
1618     if (runOctane)
1619         addTestsByGroup(OctaneGroup);
1620
1621     if (runRexBench)
1622         addTestsByGroup(RexBenchGroup);
1623
1624     if (runSeaMonster)
1625         addTestsByGroup(SeaMonsterGroup);
1626
1627     if (runSimple)
1628         addTestsByGroup(SimpleGroup);
1629
1630     if (runSunSpider)
1631         addTestsByGroup(SunSpiderGroup);
1632
1633     if (runWasm)
1634         addTestsByGroup(WasmGroup);
1635
1636     if (runWorkerTests)
1637         addTestsByGroup(WorkerTestsGroup);
1638
1639     if (runWSL)
1640         addTestsByGroup(WSLGroup);
1641
1642     if (runWTB)
1643         addTestsByGroup(WTBGroup);
1644 }