Have a CLI version of JetStream 2 to run on watchOS
[WebKit-https.git] / PerformanceTests / JetStream2 / JetStreamDriver.js
1 "use strict";
2
3 /*
4  * Copyright (C) 2018 Apple Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
19  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
25  * THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 const preloadResources = !isInBrowser;
29 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
30
31 if (typeof RAMification === "undefined")
32     var RAMification = false;
33
34 if (typeof testIterationCount === "undefined")
35     var testIterationCount = undefined;
36
37 if (typeof testIterationCountMap === "undefined")
38     var testIterationCountMap = new Map;
39
40 if (typeof testWorstCaseCountMap === "undefined")
41     var testWorstCaseCountMap = new Map;
42
43 if (typeof dumpJSONResults === "undefined")
44     var dumpJSONResults = false;
45
46 // Used for the promise representing the current benchmark run.
47 this.currentResolve = null;
48 this.currentReject = null;
49
50 const defaultIterationCount = 120;
51 const defaultWorstCaseCount = 4;
52
53 let showScoreDetails = false;
54 let categoryScores = null;
55
56 function displayCategoryScores() {
57     if (!categoryScores)
58         return;
59
60     let summaryElement = document.getElementById("result-summary");
61     for (let [category, scores] of categoryScores)
62         summaryElement.innerHTML += `<p> ${category}: ${uiFriendlyNumber(geomean(scores))}</p>`
63
64     categoryScores = null;
65 }
66
67 function getIterationCount(plan) {
68     if (testIterationCountMap.has(plan.name))
69         return testIterationCountMap.get(plan.name);
70     if (testIterationCount)
71         return testIterationCount;
72     if (plan.iterations)
73         return plan.iterations;
74     return defaultIterationCount;
75 }
76
77 function getWorstCaseCount(plan) {
78     if (testWorstCaseCountMap.has(plan.name))
79         return testWorstCaseCountMap.get(plan.name);
80     if (plan.worstCaseCount)
81         return plan.worstCaseCount;
82     return defaultWorstCaseCount;
83 }
84
85 if (isInBrowser) {
86     document.onkeydown = (keyboardEvent) => {
87         let key = keyboardEvent.key;
88         if (key === "d" || key === "D") {
89             showScoreDetails = true;
90
91             displayCategoryScores();
92         }
93     };
94 }
95
96 function assert(b, m = "") {
97     if (!b)
98         throw new Error("Bad assertion: " + m);
99 }
100
101 function firstID(benchmark) {
102     return `results-cell-${benchmark.name}-first`;
103 }
104
105 function worst4ID(benchmark) {
106     return `results-cell-${benchmark.name}-worst4`;
107 }
108
109 function avgID(benchmark) {
110     return `results-cell-${benchmark.name}-avg`;
111 }
112
113 function scoreID(benchmark) {
114     return `results-cell-${benchmark.name}-score`;
115 }
116
117 function mean(values) {
118     assert(values instanceof Array);
119     let sum = 0;
120     for (let x of values)
121         sum += x;
122     return sum / values.length;
123 }
124
125 function geomean(values) {
126     assert(values instanceof Array);
127     let product = 1;
128     for (let x of values)
129         product *= x;
130     return product ** (1 / values.length);
131 }
132
133 function toScore(timeValue) {
134     return 5000 / timeValue;
135 }
136
137 function toTimeValue(score) {
138     return 5000 / score;
139 }
140
141 function updateUI() {
142     return new Promise((resolve) => {
143         if (isInBrowser)
144             requestAnimationFrame(() => setTimeout(resolve, 0));
145         else
146             resolve();
147     });
148 }
149
150 function uiFriendlyNumber(num) {
151     if (Number.isInteger(num))
152         return num;
153     return num.toFixed(3);
154 }
155
156 function uiFriendlyDuration(time)
157 {
158     let minutes = time.getMinutes();
159     let seconds = time.getSeconds();
160     let milliSeconds = time.getMilliseconds();
161     let result = "" + minutes + ":";
162
163     result = result + (seconds < 10 ? "0" : "") + seconds + ".";
164     result = result + (milliSeconds < 10 ? "00" : (milliSeconds < 100 ? "0" : "")) + milliSeconds;
165
166     return result;
167 }
168
169 const fileLoader = (function() {
170     class Loader {
171         constructor() {
172             this.requests = new Map;
173         }
174
175         async _loadInternal(url) {
176             if (!isInBrowser)
177                 return Promise.resolve(readFile(url));
178
179             let fetchResponse = await fetch(new Request(url));
180             if (url.indexOf(".js") !== -1)
181                 return await fetchResponse.text();
182             else if (url.indexOf(".wasm") !== -1)
183                 return await fetchResponse.arrayBuffer();
184
185             throw new Error("should not be reached!");
186         }
187
188         async load(url) {
189             if (this.requests.has(url))
190                 return (await this.requests.get(url));
191
192             let promise = this._loadInternal(url);
193             this.requests.set(url, promise);
194             return (await promise);
195         }
196     }
197     return new Loader;
198 })();
199
200 class Driver {
201     constructor() {
202         this.benchmarks = [];
203     }
204
205     addPlan(plan, BenchmarkClass = DefaultBenchmark) {
206         this.benchmarks.push(new BenchmarkClass(plan));
207     }
208
209     async start() {
210         let statusElement = false;
211         let summaryElement = false;
212         if (isInBrowser) {
213             statusElement = document.getElementById("status");
214             summaryElement = document.getElementById("result-summary");
215             statusElement.innerHTML = `<label>Running...</label>`;
216         } else {
217             console.log("Starting JetStream2");
218         }
219
220         await updateUI();
221
222         let start = Date.now();
223         for (let benchmark of this.benchmarks) {
224             benchmark.updateUIBeforeRun();
225
226             await updateUI();
227
228             try {
229                 await benchmark.run();
230             } catch(e) {
231                 JetStream.reportError(benchmark);
232                 throw e;
233             }
234
235             benchmark.updateUIAfterRun();
236         }
237
238         let totalTime = Date.now() - start;
239         if (measureTotalTimeAsSubtest) {
240             if (isInBrowser)
241                 document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime);
242             else
243                 console.log("Total time:", uiFriendlyNumber(totalTime));
244             allScores.push(totalTime);
245         }
246
247         let allScores = [];
248         for (let benchmark of this.benchmarks)
249             allScores.push(benchmark.score);
250
251         categoryScores = new Map;
252         for (let benchmark of this.benchmarks) {
253             for (let category of Object.keys(benchmark.subTimes()))
254                 categoryScores.set(category, []);
255         }
256
257         for (let benchmark of this.benchmarks) {
258             for (let [category, value] of Object.entries(benchmark.subTimes())) {
259                 let arr = categoryScores.get(category);
260                 arr.push(value);
261             }
262         }
263
264         if (isInBrowser) {
265             summaryElement.classList.add('done');
266             summaryElement.innerHTML = "<div class=\"score\">" + uiFriendlyNumber(geomean(allScores)) + "</div><label>Score</label>";
267             summaryElement.onclick = displayCategoryScores;
268             if (showScoreDetails)
269                 displayCategoryScores();
270             statusElement.innerHTML = '';
271         } else {
272             console.log("\n");
273             for (let [category, scores] of categoryScores)
274                 console.log(`${category}: ${uiFriendlyNumber(geomean(scores))}`);
275
276             console.log("\nTotal Score: ", uiFriendlyNumber(geomean(allScores)), "\n");
277         }
278
279         this.reportScoreToRunBenchmarkRunner();
280         this.dumpJSONResultsIfNeeded();
281     }
282
283     runCode(string)
284     {
285         if (!isInBrowser) {
286             let scripts = string;
287             let globalObject = runString("");
288             globalObject.console = {log:globalObject.print}
289             globalObject.top = {
290                 currentResolve,
291                 currentReject
292             };
293             for (let script of scripts)
294                 globalObject.loadString(script);
295             return globalObject;
296         }
297
298         var magic = document.getElementById("magic");
299         magic.contentDocument.body.textContent = "";
300         magic.contentDocument.body.innerHTML = "<iframe id=\"magicframe\" frameborder=\"0\">";
301
302         var magicFrame = magic.contentDocument.getElementById("magicframe");
303         magicFrame.contentDocument.open();
304         magicFrame.contentDocument.write("<!DOCTYPE html><head><title>benchmark payload</title></head><body>\n" + string + "</body></html>");
305
306         return magicFrame;
307     }
308
309     prepareToRun()
310     {
311         this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
312
313         let text = "";
314         let newBenchmarks = [];
315         for (let benchmark of this.benchmarks) {
316             let id = JSON.stringify(benchmark.constructor.scoreDescription());
317             let description = JSON.parse(id);
318
319             newBenchmarks.push(benchmark);
320             let scoreIds = benchmark.scoreIdentifiers()
321             let overallScoreId = scoreIds.pop();
322
323             if (isInBrowser) {
324                 text +=
325                     `<div class="benchmark" id="benchmark-${benchmark.name}">
326                     <h3 class="benchmark-name"><a href="in-depth.html#${benchmark.name}">${benchmark.name}</a></h3>
327                     <h4 class="score" id="${overallScoreId}">___</h4><p>`;
328                 for (let i = 0; i < scoreIds.length; i++) {
329                     let id = scoreIds[i];
330                     let label = description[i];
331                     text += `<span class="result"><span id="${id}">___</span><label>${label}</label></span>`
332                 }
333                 text += `</p></div>`;
334             }
335         }
336
337         if (!isInBrowser)
338             return;
339
340         for (let f = 0; f < 5; f++)
341             text += `<div class="benchmark fill"></div>`;
342
343         let timestamp = Date.now();
344         document.getElementById('jetstreams').style.backgroundImage = `url('jetstreams.svg?${timestamp}')`;
345         let resultsTable = document.getElementById("results");
346         resultsTable.innerHTML = text;
347
348         document.getElementById("magic").textContent = "";
349         document.addEventListener('keypress', function (e) {
350             if (e.which === 13)
351                 JetStream.start();
352         });
353     }
354
355     reportError(benchmark)
356     {
357         for (let id of benchmark.scoreIdentifiers())
358             document.getElementById(id).innerHTML = "error";
359     }
360
361     async initialize() {
362         await this.fetchResources();
363         this.prepareToRun();
364         if (isInBrowser && window.location.search == '?report=true') {
365             setTimeout(() => this.start(), 4000);
366         }
367     }
368
369     async fetchResources() {
370         for (let benchmark of this.benchmarks)
371             await benchmark.fetchResources();
372
373         if (!isInBrowser)
374             return;
375
376         let statusElement = document.getElementById("status");
377         statusElement.classList.remove('loading');
378         statusElement.innerHTML = `<a href="javascript:JetStream.start()" class="button">Start Test</a>`;
379         statusElement.onclick = () => {
380             statusElement.onclick = null;
381             JetStream.start();
382             return false;
383         }
384     }
385
386     resultsJSON()
387     {
388         let results = {};
389         for (let benchmark of this.benchmarks) {
390             const subResults = {}
391             const subTimes = benchmark.subTimes();
392             for (const name in subTimes) {
393                 subResults[name] = {"metrics": {"Time": {"current": [toTimeValue(subTimes[name])]}}};
394             }
395             results[benchmark.name] = {
396                 "metrics" : {
397                     "Score" : {"current" : [benchmark.score]},
398                     "Time": ["Geometric"],
399                 },
400                 "tests": subResults,
401             };
402         }
403
404         results = {"JetStream2.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
405
406         return JSON.stringify(results);
407     }
408
409     dumpJSONResultsIfNeeded()
410     {
411         if (dumpJSONResults) {
412             print("\n");
413             print(this.resultsJSON());
414             print("\n");
415         }
416     }
417
418     async reportScoreToRunBenchmarkRunner()
419     {
420         if (!isInBrowser)
421             return;
422
423         if (window.location.search !== '?report=true')
424             return;
425
426         const content = this.resultsJSON();
427         await fetch("/report", {
428             method: "POST",
429             heeaders: {
430                 "Content-Type": "application/json",
431                 "Content-Length": content.length,
432                 "Connection": "close",
433             },
434             body: content,
435         });
436     }
437 };
438
439 class Benchmark {
440     constructor(plan)
441     {
442         this.plan = plan;
443         this.iterations = getIterationCount(plan);
444         this.isAsync = !!plan.isAsync;
445
446         this.scripts = null;
447
448         this._resourcesPromise = null;
449         this.fetchResources();
450     }
451
452     get name() { return this.plan.name; }
453
454     get runnerCode() {
455         return `
456             let __benchmark = new Benchmark(${this.iterations});
457             let results = [];
458             for (let i = 0; i < ${this.iterations}; i++) {
459                 if (__benchmark.prepareForNextIteration)
460                     __benchmark.prepareForNextIteration();
461
462                 let start = Date.now();
463                 __benchmark.runIteration();
464                 let end = Date.now();
465
466                 results.push(Math.max(1, end - start));
467             }
468             if (__benchmark.validate)
469                 __benchmark.validate();
470             top.currentResolve(results);`;
471     }
472
473     processResults() {
474         throw new Error("Subclasses need to implement this");
475     }
476
477     get score() {
478         throw new Error("Subclasses need to implement this");
479     }
480
481     get prerunCode() { return null; }
482
483     async run() {
484         let code;
485         if (isInBrowser)
486             code = "";
487         else
488             code = [];
489
490         let addScript = (text) => {
491             if (isInBrowser)
492                 code += `<script>${text}</script>`;
493             else
494                 code.push(text);
495         };
496
497         let addScriptWithURL = (url) => {
498             if (isInBrowser)
499                 code += `<script src="${url}"></script>`;
500             else
501                 assert(false, "Should not reach here in CLI");
502         };
503
504         addScript(`const isInBrowser = ${isInBrowser}; let performance = {now: Date.now.bind(Date)};`);
505
506         if (!!this.plan.deterministicRandom) {
507             addScript(`
508                 Math.random = (function() {
509                     var seed = 49734321;
510                     return function() {
511                         // Robert Jenkins' 32 bit integer hash function.
512                         seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
513                         seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
514                         seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
515                         seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
516                         seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
517                         seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
518                         return (seed & 0xfffffff) / 0x10000000;
519                     };
520                 })();
521             `);
522
523         }
524
525         if (this.plan.preload) {
526             let str = "";
527             for (let [variableName, blobUrl] of this.preloads)
528                 str += `const ${variableName} = "${blobUrl}";\n`;
529             addScript(str);
530         }
531
532         let prerunCode = this.prerunCode;
533         if (prerunCode)
534             addScript(prerunCode);
535
536         if (preloadResources) {
537             assert(this.scripts && this.scripts.length === this.plan.files.length);
538
539             for (let text of this.scripts)
540                 addScript(text);
541         } else {
542             for (let file of this.plan.files)
543                 addScriptWithURL(file);
544         }
545
546         let promise = new Promise((resolve, reject) => {
547             currentResolve = resolve;
548             currentReject = reject;
549         });
550
551         if (isInBrowser) {
552             code = `
553                 <script> window.onerror = top.currentReject; </script>
554                 ${code}
555             `;
556         }
557         addScript(this.runnerCode);
558
559         this.startTime = new Date();
560
561         if (RAMification)
562             resetMemoryPeak();
563
564         let magicFrame;
565         try {
566             magicFrame = JetStream.runCode(code);
567         } catch(e) {
568             console.log("Error in runCode: ", e);
569             throw e;
570         }
571         let results = await promise;
572
573         this.endTime = new Date();
574
575         if (RAMification) {
576             let memoryFootprint = MemoryFootprint();
577             this.currentFootprint = memoryFootprint.current;
578             this.peakFootprint = memoryFootprint.peak;
579         }
580
581         this.processResults(results);
582         if (isInBrowser)
583             magicFrame.contentDocument.close();
584     }
585
586     fetchResources() {
587         if (this._resourcesPromise)
588             return this._resourcesPromise;
589
590         let filePromises = preloadResources ? this.plan.files.map((file) => fileLoader.load(file)) : [];
591         let preloads = [];
592         let preloadVariableNames = [];
593
594         if (isInBrowser && this.plan.preload) {
595             for (let prop of Object.getOwnPropertyNames(this.plan.preload)) {
596                 preloadVariableNames.push(prop);
597                 preloads.push(this.plan.preload[prop]);
598             }
599         }
600
601         preloads = preloads.map((file) => fileLoader.load(file));
602
603         let p1 = Promise.all(filePromises).then((texts) => {
604             if (!preloadResources)
605                 return;
606             this.scripts = [];
607             assert(texts.length === this.plan.files.length);
608             for (let text of texts)
609                 this.scripts.push(text);
610         });
611
612         let p2 = Promise.all(preloads).then((data) => {
613             this.preloads = [];
614             this.blobs = [];
615             for (let i = 0; i < data.length; ++i) {
616                 let item = data[i];
617
618                 let blob;
619                 if (typeof item === "string") {
620                     blob = new Blob([item], {type : 'application/javascript'});
621                 } else if (item instanceof ArrayBuffer) {
622                     blob = new Blob([item], {type : 'application/octet-stream'});
623                 } else
624                     throw new Error("Unexpected item!");
625
626                 this.blobs.push(blob);
627                 this.preloads.push([preloadVariableNames[i], URL.createObjectURL(blob)]);
628             }
629         });
630
631         this._resourcesPromise = Promise.all([p1, p2]);
632         return this._resourcesPromise;
633     }
634
635     static scoreDescription() { throw new Error("Must be implemented by subclasses."); }
636     scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); }
637
638     updateUIBeforeRun() {
639         if (!isInBrowser) {
640             console.log(`Running ${this.name}:`);
641             return;
642         }
643
644         let containerUI = document.getElementById("results");
645         let resultsBenchmarkUI = document.getElementById(`benchmark-${this.name}`);
646         containerUI.insertBefore(resultsBenchmarkUI, containerUI.firstChild);
647         resultsBenchmarkUI.classList.add("benchmark-running");
648
649         for (let id of this.scoreIdentifiers())
650             document.getElementById(id).innerHTML = "...";
651     }
652
653     updateUIAfterRun() {
654         if (!isInBrowser)
655             return;
656
657         let benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`);
658         benchmarkResultsUI.classList.remove("benchmark-running");
659         benchmarkResultsUI.classList.add("benchmark-done");
660
661     }
662 };
663
664 class DefaultBenchmark extends Benchmark {
665     constructor(...args) {
666         super(...args);
667
668         this.worstCaseCount = getWorstCaseCount(this.plan);
669         this.firstIteration = null;
670         this.worst4 = null;
671         this.average = null;
672     }
673
674     processResults(results) {
675         function copyArray(a) {
676             let result = [];
677             for (let x of a)
678                 result.push(x);
679             return result;
680         }
681         results = copyArray(results);
682
683         this.firstIteration = toScore(results[0]);
684
685         results = results.slice(1);
686         results.sort((a, b) => a < b ? 1 : -1);
687         for (let i = 0; i + 1 < results.length; ++i)
688             assert(results[i] >= results[i + 1]);
689
690         let worstCase = [];
691         for (let i = 0; i < this.worstCaseCount; ++i)
692             worstCase.push(results[i]);
693         this.worst4 = toScore(mean(worstCase));
694         this.average = toScore(mean(results));
695     }
696
697     get score() {
698         return geomean([this.firstIteration, this.worst4, this.average]);
699     }
700
701     subTimes() {
702         return {
703             "First": this.firstIteration,
704             "Worst": this.worst4,
705             "Average": this.average,
706         };
707     }
708
709     static scoreDescription() {
710         return ["First", "Worst", "Average", "Score"];
711     }
712
713     scoreIdentifiers() {
714         return [firstID(this), worst4ID(this), avgID(this), scoreID(this)];
715     }
716
717     updateUIAfterRun() {
718         super.updateUIAfterRun();
719
720         if (isInBrowser) {
721             document.getElementById(firstID(this)).innerHTML = uiFriendlyNumber(this.firstIteration);
722             document.getElementById(worst4ID(this)).innerHTML = uiFriendlyNumber(this.worst4);
723             document.getElementById(avgID(this)).innerHTML = uiFriendlyNumber(this.average);
724             document.getElementById(scoreID(this)).innerHTML = uiFriendlyNumber(this.score);
725             return;
726         }
727
728         print("    Startup:", uiFriendlyNumber(this.firstIteration));
729         print("    Worst Case:", uiFriendlyNumber(this.worst4));
730         print("    Average:", uiFriendlyNumber(this.average));
731         print("    Score:", uiFriendlyNumber(this.score));
732         if (RAMification) {
733             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
734             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
735         }
736         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
737     }
738 }
739
740 class AsyncBenchmark extends DefaultBenchmark {
741     get runnerCode() {
742         return `
743         async function doRun() {
744             let __benchmark = new Benchmark();
745             let results = [];
746             for (let i = 0; i < ${this.iterations}; i++) {
747                 let start = Date.now();
748                 await __benchmark.runIteration();
749                 let end = Date.now();
750                 results.push(Math.max(1, end - start));
751             }
752             if (__benchmark.validate)
753                 __benchmark.validate();
754             top.currentResolve(results);
755         }
756         doRun();`
757     }
758 };
759
760 class WSLBenchmark extends Benchmark {
761     constructor(...args) {
762         super(...args);
763
764         this.stdlib = null;
765         this.mainRun = null;
766     }
767
768     processResults(results) {
769         this.stdlib = toScore(results[0]);
770         this.mainRun = toScore(results[1]);
771     }
772
773     get score() {
774         return geomean([this.stdlib, this.mainRun]);
775     }
776
777     get runnerCode() {
778         return `
779             let benchmark = new Benchmark();
780             let results = [];
781             {
782                 let start = Date.now();
783                 benchmark.buildStdlib();
784                 results.push(Date.now() - start);
785             }
786
787             {
788                 let start = Date.now();
789                 benchmark.run();
790                 results.push(Date.now() - start);
791             }
792
793             top.currentResolve(results);
794             `;
795     }
796
797     subTimes() {
798         return {
799             "Stdlib": this.stdlib,
800             "MainRun": this.mainRun,
801         };
802     }
803
804     static scoreDescription() {
805         return ["Stdlib", "MainRun", "Score"];
806     }
807
808     scoreIdentifiers() {
809         return ["wsl-stdlib-score", "wsl-tests-score", "wsl-score-score"];
810     }
811
812     updateUIAfterRun() {
813         super.updateUIAfterRun();
814
815         if (isInBrowser) {
816             document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyNumber(this.stdlib);
817             document.getElementById("wsl-tests-score").innerHTML = uiFriendlyNumber(this.mainRun);
818             document.getElementById("wsl-score-score").innerHTML = uiFriendlyNumber(this.score);
819             return;
820         }
821
822         print("    Stdlib:", uiFriendlyNumber(this.stdlib));
823         print("    Tests:", uiFriendlyNumber(this.mainRun));
824         print("    Score:", uiFriendlyNumber(this.score));
825         if (RAMification) {
826             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
827             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
828         }
829         print("    Wall time:", uiFriendlyDuration(new Date(this.endTime - this.startTime)));
830     }
831 };
832
833 class WasmBenchmark extends Benchmark {
834     constructor(...args) {
835         super(...args);
836
837         this.startupTime = null;
838         this.runTime = null;
839     }
840
841     processResults(results) {
842         this.startupTime = toScore(results[0]);
843         this.runTime = toScore(results[1]);
844     }
845
846     get score() {
847         return geomean([this.startupTime, this.runTime]);
848     }
849
850     get wasmPath() {
851         return this.plan.wasmPath;
852     }
853
854     get prerunCode() {
855         let str = `
856             let verbose = false;
857
858             let compileTime = null;
859             let runTime = null;
860
861             let globalObject = this;
862
863             globalObject.benchmarkTime = Date.now.bind(Date);
864
865             globalObject.reportCompileTime = (t) => {
866                 if (compileTime !== null)
867                     throw new Error("called report compile time twice");
868                 compileTime = t;
869             };
870
871             globalObject.reportRunTime = (t) => {
872                 if (runTime !== null)
873                     throw new Error("called report run time twice")
874                 runTime = t;
875                 top.currentResolve([compileTime, runTime]);
876             };
877
878             abort = quit = function() {
879                 if (verbose)
880                     console.log('Intercepted quit/abort');
881             };
882
883             oldPrint = globalObject.print;
884             globalObject.print = globalObject.printErr = (...args) => {
885                 if (verbose)
886                     console.log('Intercepted print: ', ...args);
887             };
888
889             let Module = {
890                 preRun: [],
891                 postRun: [],
892                 print: function() { },
893                 printErr: function() { },
894                 setStatus: function(text) {
895                 },
896                 totalDependencies: 0,
897                 monitorRunDependencies: function(left) {
898                     this.totalDependencies = Math.max(this.totalDependencies, left);
899                     Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
900                 }
901             };
902             globalObject.Module = Module;
903             `;
904         return str;
905     }
906
907     get runnerCode() {
908         let str = "";
909         if (isInBrowser) {
910             str += `
911                 var xhr = new XMLHttpRequest();
912                 xhr.open('GET', wasmBlobURL, true);
913                 xhr.responseType = 'arraybuffer';
914                 xhr.onload = function() {
915                     Module.wasmBinary = xhr.response;
916                     doRun();
917                 };
918                 xhr.send(null);
919             `;
920         } else {
921             str += `
922             Module.wasmBinary = read("${this.wasmPath}", "binary");
923             globalObject.read = (...args) => {
924                 console.log("should not be inside read: ", ...args);
925                 throw new Error;
926             };
927
928             Module.setStatus = null;
929             Module.monitorRunDependencies = null;
930
931             Promise.resolve(42).then(() => {
932                 try {
933                     doRun();
934                 } catch(e) {
935                     console.log("error running wasm:", e);
936                     throw e;
937                 }
938             })
939             `;
940         }
941         return str;
942     }
943
944     subTimes() {
945         return {
946             "Startup": this.startupTime,
947             "Runtime": this.runTime,
948         };
949     }
950
951     static scoreDescription() {
952         return ["Startup", "Runtime", "Score"];
953     }
954
955     get startupID() {
956         return `wasm-startup-id${this.name}`;
957     }
958     get runID() {
959         return `wasm-run-id${this.name}`;
960     }
961     get scoreID() {
962         return `wasm-score-id${this.name}`;
963     }
964
965     scoreIdentifiers() {
966         return [this.startupID, this.runID, this.scoreID];
967     }
968
969     updateUIAfterRun() {
970         super.updateUIAfterRun();
971
972         if (isInBrowser) {
973             document.getElementById(this.startupID).innerHTML = uiFriendlyNumber(this.startupTime);
974             document.getElementById(this.runID).innerHTML = uiFriendlyNumber(this.runTime);
975             document.getElementById(this.scoreID).innerHTML = uiFriendlyNumber(this.score);
976             return;
977         }
978         print("    Startup:", uiFriendlyNumber(this.startupTime));
979         print("    Run time:", uiFriendlyNumber(this.runTime));
980         if (RAMification) {
981             print("    Current Footprint:", uiFriendlyNumber(this.currentFootprint));
982             print("    Peak Footprint:", uiFriendlyNumber(this.peakFootprint));
983         }
984         print("    Score:", uiFriendlyNumber(this.score));
985     }
986 };
987
988 const ARESGroup = Symbol.for("ARES");
989 const CDJSGroup = Symbol.for("CDJS");
990 const CodeLoadGroup = Symbol.for("CodeLoad");
991 const LuaJSFightGroup = Symbol.for("LuaJSFight");
992 const OctaneGroup = Symbol.for("Octane");
993 const RexBenchGroup = Symbol.for("RexBench");
994 const SeaMonsterGroup = Symbol.for("SeaMonster");
995 const SimpleGroup = Symbol.for("Simple");
996 const SunSpiderGroup = Symbol.for("SunSpider");
997 const WasmGroup = Symbol.for("Wasm");
998 const WorkerTestsGroup = Symbol.for("WorkerTests");
999 const WSLGroup = Symbol.for("WSL");
1000 const WTBGroup = Symbol.for("WTB");
1001
1002
1003 let testPlans = [
1004     // ARES
1005     {
1006         name: "Air",
1007         files: [
1008             "./ARES-6/Air/symbols.js"
1009             , "./ARES-6/Air/tmp_base.js"
1010             , "./ARES-6/Air/arg.js"
1011             , "./ARES-6/Air/basic_block.js"
1012             , "./ARES-6/Air/code.js"
1013             , "./ARES-6/Air/frequented_block.js"
1014             , "./ARES-6/Air/inst.js"
1015             , "./ARES-6/Air/opcode.js"
1016             , "./ARES-6/Air/reg.js"
1017             , "./ARES-6/Air/stack_slot.js"
1018             , "./ARES-6/Air/tmp.js"
1019             , "./ARES-6/Air/util.js"
1020             , "./ARES-6/Air/custom.js"
1021             , "./ARES-6/Air/liveness.js"
1022             , "./ARES-6/Air/insertion_set.js"
1023             , "./ARES-6/Air/allocate_stack.js"
1024             , "./ARES-6/Air/payload-gbemu-executeIteration.js"
1025             , "./ARES-6/Air/payload-imaging-gaussian-blur-gaussianBlur.js"
1026             , "./ARES-6/Air/payload-airjs-ACLj8C.js"
1027             , "./ARES-6/Air/payload-typescript-scanIdentifier.js"
1028             , "./ARES-6/Air/benchmark.js"
1029         ],
1030         testGroup: ARESGroup
1031     },
1032     {
1033         name: "Basic",
1034         files: [
1035             "./ARES-6/Basic/ast.js"
1036             , "./ARES-6/Basic/basic.js"
1037             , "./ARES-6/Basic/caseless_map.js"
1038             , "./ARES-6/Basic/lexer.js"
1039             , "./ARES-6/Basic/number.js"
1040             , "./ARES-6/Basic/parser.js"
1041             , "./ARES-6/Basic/random.js"
1042             , "./ARES-6/Basic/state.js"
1043             , "./ARES-6/Basic/util.js"
1044             , "./ARES-6/Basic/benchmark.js"
1045         ],
1046         testGroup: ARESGroup
1047     },
1048     {
1049         name: "ML",
1050         files: [
1051             "./ARES-6/ml/index.js"
1052             , "./ARES-6/ml/benchmark.js"
1053         ],
1054         iterations: 60,
1055         testGroup: ARESGroup
1056     },
1057     {
1058         name: "Babylon",
1059         files: [
1060             "./ARES-6/Babylon/index.js"
1061             , "./ARES-6/Babylon/benchmark.js"
1062         ],
1063         preload: {
1064             airBlob: "./ARES-6/Babylon/air-blob.js",
1065             basicBlob: "./ARES-6/Babylon/basic-blob.js",
1066             inspectorBlob: "./ARES-6/Babylon/inspector-blob.js",
1067             babylonBlob: "./ARES-6/Babylon/babylon-blob.js"
1068         },
1069         testGroup: ARESGroup
1070     },
1071     // CDJS
1072     {
1073         name: "cdjs",
1074         files: [
1075             "./cdjs/constants.js"
1076             , "./cdjs/util.js"
1077             , "./cdjs/red_black_tree.js"
1078             , "./cdjs/call_sign.js"
1079             , "./cdjs/vector_2d.js"
1080             , "./cdjs/vector_3d.js"
1081             , "./cdjs/motion.js"
1082             , "./cdjs/reduce_collision_set.js"
1083             , "./cdjs/simulator.js"
1084             , "./cdjs/collision.js"
1085             , "./cdjs/collision_detector.js"
1086             , "./cdjs/benchmark.js"
1087         ],
1088         iterations: 60,
1089         worstCaseCount: 3,
1090         testGroup: CDJSGroup
1091     },
1092     // CodeLoad
1093     {
1094         name: "first-inspector-code-load",
1095         files: [
1096             "./code-load/code-first-load.js"
1097         ],
1098         preload: {
1099             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1100         },
1101         testGroup: CodeLoadGroup
1102     },
1103     {
1104         name: "multi-inspector-code-load",
1105         files: [
1106             "./code-load/code-multi-load.js"
1107         ],
1108         preload: {
1109             inspectorPayloadBlob: "./code-load/inspector-payload-minified.js"
1110         },
1111         testGroup: CodeLoadGroup
1112     },
1113     // Octane
1114     {
1115         name: "Box2D",
1116         files: [
1117             "./Octane/box2d.js"
1118         ],
1119         deterministicRandom: true,
1120         testGroup: OctaneGroup
1121     },
1122     {
1123         name: "octane-code-load",
1124         files: [
1125             "./Octane/code-first-load.js"
1126         ],
1127         deterministicRandom: true,
1128         testGroup: OctaneGroup
1129     },
1130     {
1131         name: "crypto",
1132         files: [
1133             "./Octane/crypto.js"
1134         ],
1135         deterministicRandom: true,
1136         testGroup: OctaneGroup
1137     },
1138     {
1139         name: "delta-blue",
1140         files: [
1141             "./Octane/deltablue.js"
1142         ],
1143         deterministicRandom: true,
1144         testGroup: OctaneGroup
1145     },
1146     {
1147         name: "earley-boyer",
1148         files: [
1149             "./Octane/earley-boyer.js"
1150         ],
1151         deterministicRandom: true,
1152         testGroup: OctaneGroup
1153     },
1154     {
1155         name: "gbemu",
1156         files: [
1157             "./Octane/gbemu-part1.js"
1158             , "./Octane/gbemu-part2.js"
1159         ],
1160         deterministicRandom: true,
1161         testGroup: OctaneGroup
1162     },
1163     {
1164         name: "mandreel",
1165         files: [
1166             "./Octane/mandreel.js"
1167         ],
1168         iterations: 80,
1169         deterministicRandom: true,
1170         testGroup: OctaneGroup
1171     },
1172     {
1173         name: "navier-stokes",
1174         files: [
1175             "./Octane/navier-stokes.js"
1176         ],
1177         deterministicRandom: true,
1178         testGroup: OctaneGroup
1179     },
1180     {
1181         name: "pdfjs",
1182         files: [
1183             "./Octane/pdfjs.js"
1184         ],
1185         deterministicRandom: true,
1186         testGroup: OctaneGroup
1187     },
1188     {
1189         name: "raytrace",
1190         files: [
1191             "./Octane/raytrace.js"
1192         ],
1193         deterministicRandom: true,
1194         testGroup: OctaneGroup
1195     },
1196     {
1197         name: "regexp",
1198         files: [
1199             "./Octane/regexp.js"
1200         ],
1201         deterministicRandom: true,
1202         testGroup: OctaneGroup
1203     },
1204     {
1205         name: "richards",
1206         files: [
1207             "./Octane/richards.js"
1208         ],
1209         deterministicRandom: true,
1210         testGroup: OctaneGroup
1211     },
1212     {
1213         name: "splay",
1214         files: [
1215             "./Octane/splay.js"
1216         ],
1217         deterministicRandom: true,
1218         testGroup: OctaneGroup
1219     },
1220     {
1221         name: "typescript",
1222         files: [
1223             "./Octane/typescript-compiler.js"
1224             , "./Octane/typescript-input.js"
1225             , "./Octane/typescript.js"
1226         ],
1227         iterations: 15,
1228         worstCaseCount: 2,
1229         deterministicRandom: true,
1230         testGroup: OctaneGroup
1231     },
1232     {
1233         name: "octane-zlib",
1234         files: [
1235             "./Octane/zlib-data.js"
1236             , "./Octane/zlib.js"
1237         ],
1238         iterations: 15,
1239         worstCaseCount: 2,
1240         deterministicRandom: true,
1241         testGroup: OctaneGroup
1242     },
1243     // RexBench
1244     {
1245         name: "FlightPlanner",
1246         files: [
1247             "./RexBench/FlightPlanner/airways.js"
1248             , "./RexBench/FlightPlanner/waypoints.js"
1249             , "./RexBench/FlightPlanner/flight_planner.js"
1250             , "./RexBench/FlightPlanner/expectations.js"
1251             , "./RexBench/FlightPlanner/benchmark.js"
1252         ],
1253         testGroup: RexBenchGroup
1254     },
1255     {
1256         name: "OfflineAssembler",
1257         files: [
1258             "./RexBench/OfflineAssembler/registers.js"
1259             , "./RexBench/OfflineAssembler/instructions.js"
1260             , "./RexBench/OfflineAssembler/ast.js"
1261             , "./RexBench/OfflineAssembler/parser.js"
1262             , "./RexBench/OfflineAssembler/file.js"
1263             , "./RexBench/OfflineAssembler/LowLevelInterpreter.js"
1264             , "./RexBench/OfflineAssembler/LowLevelInterpreter32_64.js"
1265             , "./RexBench/OfflineAssembler/LowLevelInterpreter64.js"
1266             , "./RexBench/OfflineAssembler/InitBytecodes.js"
1267             , "./RexBench/OfflineAssembler/expected.js"
1268             , "./RexBench/OfflineAssembler/benchmark.js"
1269         ],
1270         iterations: 80,
1271         testGroup: RexBenchGroup
1272     },
1273     {
1274         name: "UniPoker",
1275         files: [
1276             "./RexBench/UniPoker/poker.js"
1277             , "./RexBench/UniPoker/expected.js"
1278             , "./RexBench/UniPoker/benchmark.js"
1279         ],
1280         deterministicRandom: true,
1281         testGroup: RexBenchGroup
1282     },
1283     // Simple
1284     {
1285         name: "async-fs",
1286         files: [
1287             "./simple/file-system.js"
1288         ],
1289         iterations: 40,
1290         worstCaseCount: 3,
1291         benchmarkClass: AsyncBenchmark,
1292         testGroup: SimpleGroup
1293     },
1294     {
1295         name: "float-mm.c",
1296         files: [
1297             "./simple/float-mm.c.js"
1298         ],
1299         iterations: 15,
1300         worstCaseCount: 2,
1301         testGroup: SimpleGroup
1302     },
1303     {
1304         name: "hash-map",
1305         files: [
1306             "./simple/hash-map.js"
1307         ],
1308         testGroup: SimpleGroup
1309     },
1310     // SeaMonster
1311     {
1312         name: "ai-astar",
1313         files: [
1314             "./SeaMonster/ai-astar.js"
1315         ],
1316         testGroup: SeaMonsterGroup
1317     },
1318     {
1319         name: "gaussian-blur",
1320         files: [
1321             "./SeaMonster/gaussian-blur.js"
1322         ],
1323         testGroup: SeaMonsterGroup
1324     },
1325     {
1326         name: "stanford-crypto-aes",
1327         files: [
1328             "./SeaMonster/sjlc.js"
1329             , "./SeaMonster/stanford-crypto-aes.js"
1330         ],
1331         testGroup: SeaMonsterGroup
1332     },
1333     {
1334         name: "stanford-crypto-pbkdf2",
1335         files: [
1336             "./SeaMonster/sjlc.js"
1337             , "./SeaMonster/stanford-crypto-pbkdf2.js"
1338         ],
1339         testGroup: SeaMonsterGroup
1340     },
1341     {
1342         name: "stanford-crypto-sha256",
1343         files: [
1344             "./SeaMonster/sjlc.js"
1345             , "./SeaMonster/stanford-crypto-sha256.js"
1346         ],
1347         testGroup: SeaMonsterGroup
1348     },
1349     {
1350         name: "json-stringify-inspector",
1351         files: [
1352             "./SeaMonster/inspector-json-payload.js"
1353             , "./SeaMonster/json-stringify-inspector.js"
1354         ],
1355         iterations: 20,
1356         worstCaseCount: 2,
1357         testGroup: SeaMonsterGroup
1358     },
1359     {
1360         name: "json-parse-inspector",
1361         files: [
1362             "./SeaMonster/inspector-json-payload.js"
1363             , "./SeaMonster/json-parse-inspector.js"
1364         ],
1365         iterations: 20,
1366         worstCaseCount: 2,
1367         testGroup: SeaMonsterGroup
1368     },
1369     // Wasm
1370     {
1371         name: "HashSet-wasm",
1372         wasmPath: "./wasm/HashSet.wasm",
1373         files: [
1374             "./wasm/HashSet.js"
1375         ],
1376         preload: {
1377             wasmBlobURL: "./wasm/HashSet.wasm"
1378         },
1379         benchmarkClass: WasmBenchmark,
1380         testGroup: WasmGroup
1381     },
1382     {
1383         name: "tsf-wasm",
1384         wasmPath: "./wasm/tsf.wasm",
1385         files: [
1386             "./wasm/tsf.js"
1387         ],
1388         preload: {
1389             wasmBlobURL: "./wasm/tsf.wasm"
1390         },
1391         benchmarkClass: WasmBenchmark,
1392         testGroup: WasmGroup
1393     },
1394     {
1395         name: "quicksort-wasm",
1396         wasmPath: "./wasm/quicksort.wasm",
1397         files: [
1398             "./wasm/quicksort.js"
1399         ],
1400         preload: {
1401             wasmBlobURL: "./wasm/quicksort.wasm"
1402         },
1403         benchmarkClass: WasmBenchmark,
1404         testGroup: WasmGroup
1405     },
1406     {
1407         name: "gcc-loops-wasm",
1408         wasmPath: "./wasm/gcc-loops.wasm",
1409         files: [
1410             "./wasm/gcc-loops.js"
1411         ],
1412         preload: {
1413             wasmBlobURL: "./wasm/gcc-loops.wasm"
1414         },
1415         benchmarkClass: WasmBenchmark,
1416         testGroup: WasmGroup
1417     },
1418     {
1419         name: "richards-wasm",
1420         wasmPath: "./wasm/richards.wasm",
1421         files: [
1422             "./wasm/richards.js"
1423         ],
1424         preload: {
1425             wasmBlobURL: "./wasm/richards.wasm"
1426         },
1427         benchmarkClass: WasmBenchmark,
1428         testGroup: WasmGroup
1429     },
1430     // WorkerTests
1431     {
1432         name: "bomb-workers",
1433         files: [
1434             "./worker/bomb.js"
1435         ],
1436         iterations: 80,
1437         preload: {
1438             rayTrace3D: "./worker/bomb-subtests/3d-raytrace.js"
1439             , accessNbody: "./worker/bomb-subtests/access-nbody.js"
1440             , morph3D: "./worker/bomb-subtests/3d-morph.js"
1441             , cube3D: "./worker/bomb-subtests/3d-cube.js"
1442             , accessFunnkuch: "./worker/bomb-subtests/access-fannkuch.js"
1443             , accessBinaryTrees: "./worker/bomb-subtests/access-binary-trees.js"
1444             , accessNsieve: "./worker/bomb-subtests/access-nsieve.js"
1445             , bitopsBitwiseAnd: "./worker/bomb-subtests/bitops-bitwise-and.js"
1446             , bitopsNsieveBits: "./worker/bomb-subtests/bitops-nsieve-bits.js"
1447             , controlflowRecursive: "./worker/bomb-subtests/controlflow-recursive.js"
1448             , bitops3BitBitsInByte: "./worker/bomb-subtests/bitops-3bit-bits-in-byte.js"
1449             , botopsBitsInByte: "./worker/bomb-subtests/bitops-bits-in-byte.js"
1450             , cryptoAES: "./worker/bomb-subtests/crypto-aes.js"
1451             , cryptoMD5: "./worker/bomb-subtests/crypto-md5.js"
1452             , cryptoSHA1: "./worker/bomb-subtests/crypto-sha1.js"
1453             , dateFormatTofte: "./worker/bomb-subtests/date-format-tofte.js"
1454             , dateFormatXparb: "./worker/bomb-subtests/date-format-xparb.js"
1455             , mathCordic: "./worker/bomb-subtests/math-cordic.js"
1456             , mathPartialSums: "./worker/bomb-subtests/math-partial-sums.js"
1457             , mathSpectralNorm: "./worker/bomb-subtests/math-spectral-norm.js"
1458             , stringBase64: "./worker/bomb-subtests/string-base64.js"
1459             , stringFasta: "./worker/bomb-subtests/string-fasta.js"
1460             , stringValidateInput: "./worker/bomb-subtests/string-validate-input.js"
1461             , stringTagcloud: "./worker/bomb-subtests/string-tagcloud.js"
1462             , stringUnpackCode: "./worker/bomb-subtests/string-unpack-code.js"
1463             , regexpDNA: "./worker/bomb-subtests/regexp-dna.js"
1464         },
1465         benchmarkClass: AsyncBenchmark,
1466         testGroup: WorkerTestsGroup
1467     },
1468     {
1469         name: "segmentation",
1470         files: [
1471             "./worker/segmentation.js"
1472         ],
1473         preload: {
1474             asyncTaskBlob: "./worker/async-task.js"
1475         },
1476         iterations: 36,
1477         worstCaseCount: 3,
1478         benchmarkClass: AsyncBenchmark,
1479         testGroup: WorkerTestsGroup
1480     },
1481     // WSL
1482     {
1483         name: "WSL",
1484         files: ["./WSL/Node.js" ,"./WSL/Type.js" ,"./WSL/ReferenceType.js" ,"./WSL/Value.js" ,"./WSL/Expression.js" ,"./WSL/Rewriter.js" ,"./WSL/Visitor.js" ,"./WSL/CreateLiteral.js" ,"./WSL/CreateLiteralType.js" ,"./WSL/PropertyAccessExpression.js" ,"./WSL/AddressSpace.js" ,"./WSL/AnonymousVariable.js" ,"./WSL/ArrayRefType.js" ,"./WSL/ArrayType.js" ,"./WSL/Assignment.js" ,"./WSL/AutoWrapper.js" ,"./WSL/Block.js" ,"./WSL/BoolLiteral.js" ,"./WSL/Break.js" ,"./WSL/CallExpression.js" ,"./WSL/CallFunction.js" ,"./WSL/Check.js" ,"./WSL/CheckLiteralTypes.js" ,"./WSL/CheckLoops.js" ,"./WSL/CheckRecursiveTypes.js" ,"./WSL/CheckRecursion.js" ,"./WSL/CheckReturns.js" ,"./WSL/CheckUnreachableCode.js" ,"./WSL/CheckWrapped.js" ,"./WSL/Checker.js" ,"./WSL/CloneProgram.js" ,"./WSL/CommaExpression.js" ,"./WSL/ConstexprFolder.js" ,"./WSL/ConstexprTypeParameter.js" ,"./WSL/Continue.js" ,"./WSL/ConvertPtrToArrayRefExpression.js" ,"./WSL/DereferenceExpression.js" ,"./WSL/DoWhileLoop.js" ,"./WSL/DotExpression.js" ,"./WSL/DoubleLiteral.js" ,"./WSL/DoubleLiteralType.js" ,"./WSL/EArrayRef.js" ,"./WSL/EBuffer.js" ,"./WSL/EBufferBuilder.js" ,"./WSL/EPtr.js" ,"./WSL/EnumLiteral.js" ,"./WSL/EnumMember.js" ,"./WSL/EnumType.js" ,"./WSL/EvaluationCommon.js" ,"./WSL/Evaluator.js" ,"./WSL/ExpressionFinder.js" ,"./WSL/ExternalOrigin.js" ,"./WSL/Field.js" ,"./WSL/FindHighZombies.js" ,"./WSL/FlattenProtocolExtends.js" ,"./WSL/FlattenedStructOffsetGatherer.js" ,"./WSL/FloatLiteral.js" ,"./WSL/FloatLiteralType.js" ,"./WSL/FoldConstexprs.js" ,"./WSL/ForLoop.js" ,"./WSL/Func.js" ,"./WSL/FuncDef.js" ,"./WSL/FuncInstantiator.js" ,"./WSL/FuncParameter.js" ,"./WSL/FunctionLikeBlock.js" ,"./WSL/HighZombieFinder.js" ,"./WSL/IdentityExpression.js" ,"./WSL/IfStatement.js" ,"./WSL/IndexExpression.js" ,"./WSL/InferTypesForCall.js" ,"./WSL/Inline.js" ,"./WSL/Inliner.js" ,"./WSL/InstantiateImmediates.js" ,"./WSL/IntLiteral.js" ,"./WSL/IntLiteralType.js" ,"./WSL/Intrinsics.js" ,"./WSL/LateChecker.js" ,"./WSL/Lexer.js" ,"./WSL/LexerToken.js" ,"./WSL/LiteralTypeChecker.js" ,"./WSL/LogicalExpression.js" ,"./WSL/LogicalNot.js" ,"./WSL/LoopChecker.js" ,"./WSL/MakeArrayRefExpression.js" ,"./WSL/MakePtrExpression.js" ,"./WSL/NameContext.js" ,"./WSL/NameFinder.js" ,"./WSL/NameResolver.js" ,"./WSL/NativeFunc.js" ,"./WSL/NativeFuncInstance.js" ,"./WSL/NativeType.js" ,"./WSL/NativeTypeInstance.js" ,"./WSL/NormalUsePropertyResolver.js" ,"./WSL/NullLiteral.js" ,"./WSL/NullType.js" ,"./WSL/OriginKind.js" ,"./WSL/OverloadResolutionFailure.js" ,"./WSL/Parse.js" ,"./WSL/Prepare.js" ,"./WSL/Program.js" ,"./WSL/ProgramWithUnnecessaryThingsRemoved.js" ,"./WSL/PropertyResolver.js" ,"./WSL/Protocol.js" ,"./WSL/ProtocolDecl.js" ,"./WSL/ProtocolFuncDecl.js" ,"./WSL/ProtocolRef.js" ,"./WSL/PtrType.js" ,"./WSL/ReadModifyWriteExpression.js" ,"./WSL/RecursionChecker.js" ,"./WSL/RecursiveTypeChecker.js" ,"./WSL/ResolveNames.js" ,"./WSL/ResolveOverloadImpl.js" ,"./WSL/ResolveProperties.js" ,"./WSL/ResolveTypeDefs.js" ,"./WSL/Return.js" ,"./WSL/ReturnChecker.js" ,"./WSL/ReturnException.js" ,"./WSL/StandardLibrary.js" ,"./WSL/StatementCloner.js" ,"./WSL/StructLayoutBuilder.js" ,"./WSL/StructType.js" ,"./WSL/Substitution.js" ,"./WSL/SwitchCase.js" ,"./WSL/SwitchStatement.js" ,"./WSL/SynthesizeEnumFunctions.js" ,"./WSL/SynthesizeStructAccessors.js" ,"./WSL/TrapStatement.js" ,"./WSL/TypeDef.js" ,"./WSL/TypeDefResolver.js" ,"./WSL/TypeOrVariableRef.js" ,"./WSL/TypeParameterRewriter.js" ,"./WSL/TypeRef.js" ,"./WSL/TypeVariable.js" ,"./WSL/TypeVariableTracker.js" ,"./WSL/TypedValue.js" ,"./WSL/UintLiteral.js" ,"./WSL/UintLiteralType.js" ,"./WSL/UnificationContext.js" ,"./WSL/UnreachableCodeChecker.js" ,"./WSL/VariableDecl.js" ,"./WSL/VariableRef.js" ,"./WSL/VisitingSet.js" ,"./WSL/WSyntaxError.js" ,"./WSL/WTrapError.js" ,"./WSL/WTypeError.js" ,"./WSL/WhileLoop.js" ,"./WSL/WrapChecker.js", "./WSL/Test.js"],
1485         benchmarkClass: WSLBenchmark,
1486         testGroup: WSLGroup
1487     }
1488 ];
1489
1490 // LuaJSFight tests
1491 let luaJSFightTests = [
1492     "hello_world"
1493     , "list_search"
1494     , "lists"
1495     , "string_lists"
1496 ];
1497 for (let test of luaJSFightTests) {
1498     testPlans.push({
1499         name: `${test}-LJF`,
1500         files: [
1501             `./LuaJSFight/${test}.js`
1502         ],
1503         testGroup: LuaJSFightGroup
1504     });
1505 }
1506
1507 // SunSpider tests
1508 let sunSpiderTests = [
1509     "3d-cube"
1510     , "3d-raytrace"
1511     , "base64"
1512     , "crypto-aes"
1513     , "crypto-md5"
1514     , "crypto-sha1"
1515     , "date-format-tofte"
1516     , "date-format-xparb"
1517     , "n-body"
1518     , "regex-dna"
1519     , "string-unpack-code"
1520     , "tagcloud"
1521 ];
1522 for (let test of sunSpiderTests) {
1523     testPlans.push({
1524         name: `${test}-SP`,
1525         files: [
1526             `./SunSpider/${test}.js`
1527         ],
1528         testGroup: SunSpiderGroup
1529     });
1530 }
1531
1532 // WTB (Web Tooling Benchmark) tests
1533 let WTBTests = [
1534     "acorn"
1535     , "babylon"
1536     , "chai"
1537     , "coffeescript"
1538     , "espree"
1539     , "jshint"
1540     , "lebab"
1541     , "prepack"
1542     , "uglify-js"
1543 ];
1544 for (let name of WTBTests) {
1545     testPlans.push({
1546         name: `${name}-wtb`,
1547         files: [
1548             isInBrowser ? "./web-tooling-benchmark/browser.js" : "./web-tooling-benchmark/cli.js"
1549             , `./web-tooling-benchmark/${name}.js`
1550         ],
1551         iterations: 5,
1552         worstCaseCount: 1,
1553         testGroup: WTBGroup
1554     });
1555 }
1556
1557
1558 let testsByName = new Map();
1559 let testsByGroup = new Map();
1560
1561 for (let plan of testPlans) {
1562     let testName = plan.name;
1563
1564     if (testsByName.has(plan.name))
1565         throw "Duplicate test plan with name \"" + testName + "\"";
1566     else
1567         testsByName.set(testName, plan);
1568
1569     let group = plan.testGroup;
1570
1571     if (testsByGroup.has(group))
1572         testsByGroup.get(group).push(testName);
1573     else
1574         testsByGroup.set(group, [testName]);
1575 }
1576
1577 this.JetStream = new Driver();
1578
1579 function addTestByName(testName)
1580 {
1581     let plan = testsByName.get(testName);
1582
1583     if (plan)
1584         JetStream.addPlan(plan, plan.benchmarkClass);
1585     else
1586         throw "Couldn't find test named \"" +  testName + "\"";
1587 }
1588
1589 function addTestsByGroup(group)
1590 {
1591     let testList = testsByGroup.get(group);
1592
1593     if (!testList)
1594         throw "Couldn't find test group named: \"" + Symbol.keyFor(group) + "\"";
1595
1596     for (let testName of testList)
1597         addTestByName(testName);
1598 }
1599
1600 function processTestList(testList)
1601 {
1602     let tests = [];
1603
1604     if (testList instanceof Array)
1605         tests = testList;
1606     else
1607         tests = testList.split(/[\s,]/);
1608
1609     for (let testName of tests) {
1610         let groupTest = testsByGroup.get(Symbol.for(testName));
1611
1612         if (groupTest) {
1613             for (let testName of groupTest)
1614                 addTestByName(testName);
1615         } else
1616             addTestByName(testName);
1617     }
1618 }
1619
1620 let runOctane = true;
1621 let runARES = true;
1622 let runWSL = true;
1623 let runRexBench = true;
1624 let runWTB = true;
1625 let runSunSpider = true;
1626 let runSimple = true;
1627 let runCDJS = true;
1628 let runWorkerTests = !!isInBrowser;
1629 let runSeaMonster = true;
1630 let runCodeLoad = true;
1631 let runWasm = true;
1632
1633 if (false) {
1634     runOctane = false;
1635     runARES = false;
1636     runWSL = false;
1637     runRexBench = false;
1638     runWTB = false;
1639     runSunSpider = false;
1640     runSimple = false;
1641     runCDJS = false;
1642     runWorkerTests = false;
1643     runSeaMonster = false;
1644     runCodeLoad = false;
1645     runWasm = false;
1646 }
1647
1648 if (typeof testList !== "undefined") {
1649     processTestList(testList);
1650 } else {
1651     if (runARES)
1652         addTestsByGroup(ARESGroup);
1653
1654     if (runCDJS)
1655         addTestsByGroup(CDJSGroup);
1656
1657     if (runCodeLoad)
1658         addTestsByGroup(CodeLoadGroup);
1659
1660     if (runOctane)
1661         addTestsByGroup(OctaneGroup);
1662
1663     if (runRexBench)
1664         addTestsByGroup(RexBenchGroup);
1665
1666     if (runSeaMonster)
1667         addTestsByGroup(SeaMonsterGroup);
1668
1669     if (runSimple)
1670         addTestsByGroup(SimpleGroup);
1671
1672     if (runSunSpider)
1673         addTestsByGroup(SunSpiderGroup);
1674
1675     if (runWasm)
1676         addTestsByGroup(WasmGroup);
1677
1678     if (runWorkerTests)
1679         addTestsByGroup(WorkerTestsGroup);
1680
1681     if (runWSL)
1682         addTestsByGroup(WSLGroup);
1683
1684     if (runWTB)
1685         addTestsByGroup(WTBGroup);
1686 }