1 // Copyright (C) 2014 Apple Inc. All rights reserved.
3 // Redistribution and use in source and binary forms, with or without
4 // modification, are permitted provided that the following conditions
6 // 1. Redistributions of source code must retain the above copyright
7 // notice, this list of conditions and the following disclaimer.
8 // 2. Redistributions in binary form must reproduce the above copyright
9 // notice, this list of conditions and the following disclaimer in the
10 // documentation and/or other materials provided with the distribution.
12 // THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
13 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
14 // THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
16 // BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
17 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
18 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
19 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
20 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
22 // THE POSSIBILITY OF SUCH DAMAGE.
24 var JetStream = (function() {
29 var numberOfIterations;
34 var givenReferences = {};
39 // Import Octane benchmarks.
41 var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
42 var tMax = tDistribution.length;
49 return tDistribution[n];
52 function displayResultMessage(name, message, style)
54 var element = document.getElementById("results-cell-" + name);
55 element.innerHTML = message || "—";
56 if (element.classList) {
57 element.classList.remove("result");
58 element.classList.remove("highlighted-result");
59 element.classList.add(style);
61 element.className = style;
64 function displayResultMessageForPlan(plan, message, style)
66 for (var i = plan.benchmarks.length; i--;)
67 displayResultMessage(plan.benchmarks[i].name, message, style);
70 function computeStatistics(values)
77 for (var i = 0; i < values.length; ++i) {
84 if (n != values.length)
85 return {n: values.length, failed: values.length - n};
90 return {n: n, mean: mean};
93 for (var i = 0; i < values.length; ++i) {
96 sumForStdDev += Math.pow(values[i] - mean, 2);
98 var standardDeviation = Math.sqrt(sumForStdDev / (n - 1));
99 var standardError = standardDeviation / Math.sqrt(n);
100 var interval = tDist(n) * standardError;
101 return {n: n, mean: mean, interval: interval};
104 function formatResult(values, options)
106 options = options || {};
107 var extraPrecision = options.extraPrecision || 0;
109 function prepare(value)
111 var precision = 4 + extraPrecision;
114 var log = Math.log(value) / Math.log(10);
115 if (log >= precision)
118 digitsAfter = precision;
120 digitsAfter = precision - 1 - (log | 0);
122 digitsAfter = precision - 1;
124 return value.toFixed(digitsAfter);
127 var statistics = computeStatistics(values);
132 if ("failed" in statistics) {
133 if (statistics.n == 1)
135 return "ERROR <i>(failed " + statistics.failed + "/" + statistics.n + ")</i>";
138 if ("interval" in statistics)
139 return prepare(statistics.mean) + "<span class=\"interval\"> ± " + prepare(statistics.interval) + "</span>";
141 return prepare(statistics.mean);
144 function runCode(string)
146 var magic = document.getElementById("magic");
147 magic.contentDocument.body.textContent = "";
148 magic.contentDocument.body.innerHTML = "<iframe id=\"magicframe\" frameborder=\"0\">";
150 var magicFrame = magic.contentDocument.getElementById("magicframe");
151 magicFrame.contentDocument.open();
152 magicFrame.contentDocument.write(
153 "<!DOCTYPE html><head><title>benchmark payload</title></head><body><script>\n" +
154 "window.onerror = top.JetStream.reportError;</script>\n" +
155 string + "</body></html>");
156 magicFrame.contentDocument.close();
159 function addPlan(plan)
161 givenPlans.push(plan);
164 function addReferences(references)
166 for (var s in references)
167 givenReferences[s] = references[s];
172 var categoryMap = {};
174 for (var i = 0; i < givenPlans.length; ++i) {
175 var plan = givenPlans[i];
176 if (selectBenchmark && plan.name != selectBenchmark)
178 for (var j = 0; j < plan.benchmarks.length; ++j) {
179 var benchmark = plan.benchmarks[j];
180 benchmarks.push(benchmark);
181 var benchmarksForCategory = categoryMap[benchmark.category];
182 if (!benchmarksForCategory)
183 benchmarksForCategory = categoryMap[benchmark.category] = [];
184 benchmarksForCategory.push(benchmark);
185 benchmark.plan = plan;
186 benchmark.reference = givenReferences[benchmark.name] || 1;
190 for (var category in categoryMap)
191 categoryNames.push(category);
192 categoryNames.sort();
195 for (var i = 0; i < categoryNames.length; ++i) {
196 var categoryName = categoryNames[i];
197 cells.push({kind: "category", name: categoryName});
198 var benchmarksForCategory = categoryMap[categoryName];
199 benchmarksForCategory.sort(function(a, b) {
200 return a.name.localeCompare(b.name);
202 for (var j = 0; j < benchmarksForCategory.length; ++j)
203 cells.push({kind: "benchmark", benchmark: benchmarksForCategory[j]});
207 var remainingPlans = [].concat(givenPlans);
208 for (var i = 0; i < cells.length; ++i) {
215 var plan = cell.benchmark.plan;
216 var index = remainingPlans.indexOf(plan);
220 remainingPlans.splice(index, 1);
224 throw "Bad cell kind: " + cell.king;
229 var columnHeight = Math.ceil((cells.length + 1) / numColumns);
231 var resultsTable = document.getElementById("results");
236 for (var i = 0; i < numColumns; ++i)
237 text += "<th>Benchmark</th><th>Average Score</th>";
240 for (var i = 0; i < columnHeight; ++i) {
241 function benchmarkLine(index)
243 if (index > cells.length)
248 if (index == cells.length) {
249 result += "<td class=\"benchmark-name geometric-mean\">Geometric Mean</td>";
250 result += "<td class=\"result geometric-mean\" id=\"results-cell-geomean\">—</td>";
252 var cell = cells[index];
255 result += "<td class=\"benchmark-name category\">" + cell.name + "</td>";
256 result += "<td class=\"result category\" id=\"results-cell-geomean-" + cell.name + "\">—</td>";
260 var benchmark = cell.benchmark;
261 result += "<td class=\"benchmark-name\">";
262 result += "<a href=\"in-depth.html#" + benchmark.name + "\" target=\"_blank\">" + benchmark.name + "</a></td>";
263 result += "<td class=\"result\" id=\"results-cell-" + benchmark.name + "\">—</td>";
267 throw "Bad cell kind: " + cell.kind;
275 for (var j = 0; j < numColumns; ++j)
276 text += benchmarkLine(j * columnHeight + i);
280 resultsTable.innerHTML = text;
282 document.getElementById("magic").textContent = "";
284 for (var i = benchmarks.length; i--;) {
285 benchmarks[i].results = [];
286 benchmarks[i].times = [];
289 currentIteration = 0;
292 hasAlreadyRun = false;
295 function prepareToStart()
297 var startButton = document.getElementById("status");
298 startButton.innerHTML =
299 "<a href=\"javascript:void(JetStream.start())\">" +
300 (hasAlreadyRun ? "Test Again" : "Start Test") + "</a>";
303 function initializeWithMode(modeLine)
308 var experimentalMethod = document.getElementById("mode-description");
309 selectBenchmark = null;
310 var modes = modeLine.split(",");
311 for (var i = 0; i < modes.length; ++i) {
314 if (/benchmark=/.test(mode)) {
315 selectBenchmark = RegExp.rightContext;
319 var confidenceIntervals = "<a href=\"http://en.wikipedia.org/wiki/Confidence_interval\">confidence intervals</a>";
323 numberOfIterations = 1;
324 experimentalMethod.innerHTML =
325 "<strong>Note:</strong> Only one iteration will run per benchmark and no statistics can be calulated.<br><em>This mode is " +
326 "not statistically valid — please don't report these results.</em> <a " +
327 "href=\"javascript:JetStream.switchToNormal()\">Switch to three iterations.</a>";
331 numberOfIterations = 7;
332 experimentalMethod.innerHTML =
333 "<strong>Note:</strong> Seven iterations will run per benchmark and report scores with 95% " +
334 confidenceIntervals + ".";
338 numberOfIterations = 3;
339 experimentalMethod.textContent = "";
345 function initialize()
347 function initializeWithModeBasedOnHash()
349 initializeWithMode(window.location.hash.substr(1));
352 initializeWithModeBasedOnHash();
353 window.onpopstate = initializeWithModeBasedOnHash;
356 function switchMode(mode)
358 window.location.hash = "#" + mode;
359 initializeWithMode(mode);
364 document.getElementById("status").textContent = "";
365 document.getElementById("result-summary").textContent = "";
371 function allSelector(benchmark) { return true; }
372 function createCategorySelector(category) {
373 return function(benchmark) {
374 return benchmark.category == category;
378 function computeGeomeans(selector)
382 for (var iterationIndex = 0; ; ++iterationIndex) {
386 var allFinished = true;
387 for (var i = 0; i < benchmarks.length; ++i) {
388 if (!selector(benchmarks[i]))
391 if (iterationIndex >= benchmarks[i].results.length) {
395 if (!(iterationIndex in benchmarks[i].results))
397 sum += Math.log(benchmarks[i].results[iterationIndex]);
402 if (numDone != numSelected)
405 geomeans.push(Math.exp(sum * (1 / numDone)));
411 function formatGeomean(selector)
413 return formatResult(computeGeomeans(selector), {extraPrecision: 1});
416 function updateGeomeans()
418 for (var i = 0; i < categoryNames.length; ++i) {
419 var categoryName = categoryNames[i];
420 displayResultMessage(
421 "geomean-" + categoryName,
422 formatGeomean(createCategorySelector(categoryName)),
425 displayResultMessage(
426 "geomean", formatGeomean(allSelector),
427 computeGeomeans(allSelector).length == numberOfIterations ? "highlighted-result" : "result");
430 function computeRawResults()
432 function rawResultsLine(values) {
435 statistics: computeStatistics(values)
440 for (var i = 0; i < benchmarks.length; ++i) {
441 var line = rawResultsLine(benchmarks[i].results);
442 line.times = benchmarks[i].times;
443 line.category = benchmarks[i].category;
444 line.reference = benchmarks[i].reference;
445 rawResults[benchmarks[i].name] = line;
447 rawResults.geomean = rawResultsLine(computeGeomeans(allSelector));
454 console.log("Raw results:", JSON.stringify(computeRawResults()));
456 document.getElementById("result-summary").innerHTML = "<label>Score</label><br><span class=\"score\">" + formatGeomean(allSelector) + "</span>";
459 hasAlreadyRun = true;
469 if (currentPlan >= plans.length) {
470 if (++currentIteration >= numberOfIterations) {
477 document.getElementById("status").innerHTML = "<em>Running iteration " + (currentIteration + 1) + " of " + numberOfIterations + "\u2026</em>";
478 displayResultMessageForPlan(
479 plans[currentPlan], "<em>Running\u2026</em>", "highlighted-result");
481 accumulator = void 0;
483 window.setTimeout(function() {
486 runCode(plans[currentPlan].code);
490 function reportError(message, url, lineNumber)
492 var plan = plans[currentPlan];
495 console.error(plan.name + ": ERROR: " + url + ":" + lineNumber + ": " + message);
497 for (var i = plan.benchmarks.length; i--;) {
498 plan.benchmarks[i].times.length++;
499 plan.benchmarks[i].results.length++;
500 displayResultMessage(
501 plan.benchmarks[i].name,
502 formatResult(plan.benchmarks[i].results, plan.benchmarks[i]),
508 function reportResult()
510 var plan = plans[currentPlan];
511 for (var i = plan.benchmarks.length; i--;) {
512 var benchmark = plan.benchmarks[i];
513 benchmark.times.push(arguments[i]);
514 benchmark.results.push(100 * benchmark.reference / arguments[i]);
515 displayResultMessage(
517 formatResult(benchmark.results, plan.benchmarks[i]),
523 function accumulate(data)
526 window.setTimeout(function() {
529 runCode(plans[currentPlan].code);
533 function getAccumulator()
540 addReferences: addReferences,
541 initialize: initialize,
542 switchToQuick: function() { switchMode("quick") },
543 switchToNormal: function() { switchMode("normal") },
544 switchToLong: function() { switchMode("long") },
546 reportResult: reportResult,
547 reportError: reportError,
548 accumulate: accumulate,
549 getAccumulator: getAccumulator,