--- /dev/null
+/*
+ * Copyright (C) 2007 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+var count1 = output1.length;
+var count2 = output2.length;
+
+var itemTotals1 = {};
+itemTotals1.length = count1;
+
+var total1 = 0;
+var categoryTotals1 = {};
+var testTotalsByCategory1 = {};
+
+var mean1 = 0;
+var categoryMeans1 = {};
+var testMeansByCategory1 = {};
+
+var stdDev1 = 0;
+var categoryStdDevs1 = {};
+var testStdDevsByCategory1 = {};
+
+var stdErr1 = 0;
+var categoryStdErrs1 = {};
+var testStdErrsByCategory1 = {};
+
+var itemTotals2 = {};
+itemTotals2.length = count2;
+
+var total2 = 0;
+var categoryTotals2 = {};
+var testTotalsByCategory2 = {};
+
+var mean2 = 0;
+var categoryMeans2 = {};
+var testMeansByCategory2 = {};
+
+var stdDev2 = 0;
+var categoryStdDevs2 = {};
+var testStdDevsByCategory2 = {};
+
+var stdErr2 = 0;
+var categoryStdErrs2 = {};
+var testStdErrsByCategory2 = {};
+
+function initialize()
+{
+ itemTotals1 = {total: []};
+
+ for (var i = 0; i < categories.length; i++) {
+ var category = categories[i];
+ itemTotals1[category] = [];
+ categoryTotals1[category] = 0;
+ testTotalsByCategory1[category] = {};
+ categoryMeans1[category] = 0;
+ testMeansByCategory1[category] = {};
+ categoryStdDevs1[category] = 0;
+ testStdDevsByCategory1[category] = {};
+ categoryStdErrs1[category] = 0;
+ testStdErrsByCategory1[category] = {};
+ }
+
+ for (var i = 0; i < tests.length; i++) {
+ var test = tests[i];
+ itemTotals1[test] = [];
+ var category = test.replace(/-.*/, "");
+ testTotalsByCategory1[category][test] = 0;
+ testMeansByCategory1[category][test] = 0;
+ testStdDevsByCategory1[category][test] = 0;
+ testStdErrsByCategory1[category][test] = 0;
+ }
+
+ for (var i = 0; i < count1; i++) {
+ itemTotals1["total"][i] = 0;
+ for (var category in categoryTotals1) {
+ itemTotals1[category][i] = 0;
+ for (var test in testTotalsByCategory1[category]) {
+ itemTotals1[test][i] = 0;
+ }
+ }
+ }
+
+ itemTotals2 = {total: []};
+
+ for (var i = 0; i < categories.length; i++) {
+ var category = categories[i];
+ itemTotals2[category] = [];
+ categoryTotals2[category] = 0;
+ testTotalsByCategory2[category] = {};
+ categoryMeans2[category] = 0;
+ testMeansByCategory2[category] = {};
+ categoryStdDevs2[category] = 0;
+ testStdDevsByCategory2[category] = {};
+ categoryStdErrs2[category] = 0;
+ testStdErrsByCategory2[category] = {};
+ }
+
+ for (var i = 0; i < tests.length; i++) {
+ var test = tests[i];
+ itemTotals2[test] = [];
+ var category = test.replace(/-.*/, "");
+ testTotalsByCategory2[category][test] = 0;
+ testMeansByCategory2[category][test] = 0;
+ testStdDevsByCategory2[category][test] = 0;
+ testStdErrsByCategory2[category][test] = 0;
+ }
+
+ for (var i = 0; i < count2; i++) {
+ itemTotals2["total"][i] = 0;
+ for (var category in categoryTotals2) {
+ itemTotals2[category][i] = 0;
+ for (var test in testTotalsByCategory2[category]) {
+ itemTotals2[test][i] = 0;
+ }
+ }
+ }
+
+}
+
+function computeItemTotals(output, itemTotals)
+{
+ for (var i = 0; i < output.length; i++) {
+ var result = output[i];
+ for (var test in result) {
+ var time = result[test];
+ var category = test.replace(/-.*/, "");
+ itemTotals["total"][i] += time;
+ itemTotals[category][i] += time;
+ itemTotals[test][i] += time;
+ }
+ }
+}
+
+function computeTotals(output, categoryTotals, testTotalsByCategory)
+{
+ var total = 0;
+
+ for (var i = 0; i < output.length; i++) {
+ var result = output[i];
+ for (var test in result) {
+ var time = result[test];
+ var category = test.replace(/-.*/, "");
+ total += time;
+ categoryTotals[category] += time;
+ testTotalsByCategory[category][test] += time;
+ }
+ }
+
+ return total;
+}
+
+function computeMeans(count, total, categoryTotals, categoryMeans, testTotalsByCategory, testMeansByCategory)
+{
+ var mean = total / count;
+ for (var category in categoryTotals) {
+ categoryMeans[category] = categoryTotals[category] / count;
+ for (var test in testTotalsByCategory[category]) {
+ testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count;
+ }
+ }
+ return mean;
+}
+
+function standardDeviation(mean, items)
+{
+ var deltaSquaredSum = 0;
+ for (var i = 0; i < items.length; i++) {
+ var delta = items[i] - mean;
+ deltaSquaredSum += delta * delta;
+ }
+ variance = deltaSquaredSum / (items.length - 1);
+ return Math.sqrt(variance);
+}
+
+function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, testStdDevsByCategory, testMeansByCategory)
+{
+ var stdDev = standardDeviation(mean, itemTotals["total"]);
+ for (var category in categoryStdDevs) {
+ categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]);
+ }
+ for (var category in categoryStdDevs) {
+ for (var test in testStdDevsByCategory[category]) {
+ testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]);
+ }
+ }
+ return stdDev;
+}
+
+function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, testStdErrsByCategory, testStdDevsByCategory)
+{
+ var sqrtCount = Math.sqrt(count);
+
+ var stdErr = stdDev / sqrtCount;
+ for (var category in categoryStdErrs) {
+ categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
+ }
+ for (var category in categoryStdDevs) {
+ for (var test in testStdErrsByCategory[category]) {
+ testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount;
+ }
+ }
+
+ return stdErr;
+}
+
+var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
+var tMax = tDistribution.length;
+var tLimit = 1.96;
+
+function tDist(n)
+{
+ if (n > tMax)
+ return tLimit;
+ return tDistribution[n];
+}
+
+
+function formatMean(meanWidth, mean, stdErr, count)
+{
+ var meanString = mean.toFixed(1).toString();
+ while (meanString.length < meanWidth) {
+ meanString = " " + meanString;
+ }
+
+ var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% ";
+
+ return meanString + "ms " + error;
+}
+
+function computeLabelWidth()
+{
+ var width = "Total".length;
+ for (var category in categoryMeans1) {
+ if (category.length + 2 > width)
+ width = category.length + 2;
+ }
+ for (var i = 0; i < tests.length; i++) {
+ var shortName = tests[i].replace(/^[^-]*-/, "");
+ if (shortName.length + 4 > width)
+ width = shortName.length + 4;
+ }
+
+ return width;
+}
+
+function computeMeanWidth(mean, categoryMeans, testMeansByCategory)
+{
+ var width = mean.toFixed(1).toString().length;
+ for (var category in categoryMeans) {
+ var candidate = categoryMeans[category].toFixed(1).toString().length;
+ if (candidate > width)
+ width = candidate;
+ for (var test in testMeansByCategory[category]) {
+ var candidate = testMeansByCategory[category][test].toFixed(1).toString().length;
+ if (candidate > width)
+ width = candidate;
+ }
+ }
+
+ return width;
+}
+
+function pad(str, n)
+{
+ while (str.length < n) {
+ str += " ";
+ }
+ return str;
+}
+
+function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2)
+{
+ result = pad("", indent);
+ result += label + ": ";
+ result = pad(result, labelWidth + 2);
+
+ var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr1 * stdErr2)));
+ var df = count1 + count2 - 2;
+
+ var statisticallySignificant = (Math.abs(t) > tDist(df+1));
+ var diff = mean2 - mean1;
+ var percentage = 100 * diff / mean1;
+ var isFaster = diff < 0;
+ var probablySame = (percentage < 0.1) && !statisticallySignificant;
+
+ var diffSummary;
+ var diffDetail;
+
+ if (probablySame) {
+ diffSummary = "-";
+ diffDetail = "";
+ } else if (!statisticallySignificant) {
+ diffSummary = "??";
+ diffDetail = " not conclusive: might be " + Math.abs(percentage).toFixed(1).toString() + "% ";
+ diffDetail += (isFaster ? "faster" : "*slower*");
+ } else {
+ diffSummary = Math.abs(percentage).toFixed(1).toString() + "% " + (isFaster ? "faster" : "*slower*");
+ diffDetail = " significant";
+ }
+
+ return result + pad(diffSummary, 16) + formatMean(meanWidth1, mean1, stdErr1, count1) + " " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDetail;
+}
+
+function printOutput()
+{
+ var labelWidth = computeLabelWidth();
+ var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCategory1);
+ var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCategory2);
+
+ print("\n");
+ var header = "TEST";
+ while (header.length < labelWidth)
+ header += " ";
+ header += " COMPARISON FROM TO DETAILS";
+ print(header);
+ print("");
+ print("=============================================================================");
+ print("");
+ print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2));
+ print("");
+ print("=============================================================================");
+
+ for (var category in categoryMeans1) {
+ print("");
+ print(resultLine(labelWidth, 2, category,
+ meanWidth1, categoryMeans1[category], categoryStdErrs1[category],
+ meanWidth2, categoryMeans2[category], categoryStdErrs2[category]));
+ for (var test in testMeansByCategory1[category]) {
+ var shortName = test.replace(/^[^-]*-/, "");
+ print(resultLine(labelWidth, 4, shortName,
+ meanWidth1, testMeansByCategory1[category][test], testStdErrsByCategory1[category][test],
+ meanWidth2, testMeansByCategory2[category][test], testStdErrsByCategory2[category][test]));
+ }
+ }
+}
+
+initialize();
+
+computeItemTotals(output1, itemTotals1);
+computeItemTotals(output2, itemTotals2);
+
+total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1);
+total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2);
+
+mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTotalsByCategory1, testMeansByCategory1);
+mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTotalsByCategory2, testMeansByCategory2);
+
+stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans1, testStdDevsByCategory1, testMeansByCategory1);
+stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans2, testStdDevsByCategory2, testMeansByCategory2);
+
+stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDevs1, testStdErrsByCategory1, testStdDevsByCategory1);
+stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDevs2, testStdErrsByCategory2, testStdDevsByCategory2);
+
+printOutput();