Reviewed by Darin.

        - Add a compare script to compare results, and improve formatting of standard results a bit.

        * resources/sunspider-analyze-results.js:
        * resources/sunspider-compare-results.js: Added.
        * sunspider-compare-results: Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@26926 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/SunSpider/ChangeLog b/SunSpider/ChangeLog
index de0cdb4..5f698d9 100644 (file)
--- a/SunSpider/ChangeLog
+++ b/SunSpider/ChangeLog
@@ -1,6 +1,16 @@
 2007-10-23  Maciej Stachowiak  <mjs@apple.com>
 
         Reviewed by Darin.
+        
+        - Add a compare script to compare results, and improve formatting of standard results a bit.
+
+        * resources/sunspider-analyze-results.js:
+        * resources/sunspider-compare-results.js: Added.
+        * sunspider-compare-results: Added.
+
+2007-10-23  Maciej Stachowiak  <mjs@apple.com>
+
+        Reviewed by Darin.
 
         * resources/sunspider-analyze-results.js: Do the error range
         properly, using the t-distribution instead of 1.96 (which was

diff --git a/SunSpider/resources/sunspider-analyze-results.js b/SunSpider/resources/sunspider-analyze-results.js
index 1b5e4f2..c235b72 100644 (file)
--- a/SunSpider/resources/sunspider-analyze-results.js
+++ b/SunSpider/resources/sunspider-analyze-results.js
@@ -128,7 +128,7 @@ function standardDeviation(mean, items)
         var delta = items[i] - mean;
         deltaSquaredSum += delta * delta;
     }
-    variance = deltaSquaredSum / items.length;
+    variance = deltaSquaredSum / (items.length - 1);
     return Math.sqrt(variance);
 }
 
@@ -180,7 +180,7 @@ function formatResult(meanWidth, mean, stdErr, n)
         meanString = " " + meanString;
     }
 
-    return meanString + "ms " + "[ +/- " + (tDist(n) * stdErr).toFixed(2) + "ms | +/- " + ((tDist(n) * stdErr / mean) * 100).toFixed(2) + "% ]";
+    return meanString + "ms +/- " + ((tDist(n) * stdErr / mean) * 100).toFixed(1) + "%";
 }
 
 function computeLabelWidth()
@@ -244,6 +244,7 @@ function printOutput()
     print(resultLine(labelWidth, 0, "Total", meanWidth, mean, stdErr));
     print("----------------------------------------");
     for (var category in categoryMeans) {
+        print("");
         print(resultLine(labelWidth, 2, category, meanWidth, categoryMeans[category], categoryStdErrs[category]));
         for (var test in testMeansByCategory[category]) {
             var shortName = test.replace(/^[^-]*-/, "");

diff --git a/SunSpider/resources/sunspider-compare-results.js b/SunSpider/resources/sunspider-compare-results.js
new file mode 100644 (file)
index 0000000..47ef678
--- /dev/null
+++ b/SunSpider/resources/sunspider-compare-results.js
@@ -0,0 +1,375 @@
+/*
+ * Copyright (C) 2007 Apple Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+var count1 = output1.length;
+var count2 = output2.length;
+
+var itemTotals1 = {};
+itemTotals1.length = count1;
+
+var total1 = 0;
+var categoryTotals1 = {};
+var testTotalsByCategory1 = {};
+
+var mean1 = 0;
+var categoryMeans1 = {};
+var testMeansByCategory1 = {};
+
+var stdDev1 = 0;
+var categoryStdDevs1 = {};
+var testStdDevsByCategory1 = {};
+
+var stdErr1 = 0;
+var categoryStdErrs1 = {};
+var testStdErrsByCategory1 = {};
+
+var itemTotals2 = {};
+itemTotals2.length = count2;
+
+var total2 = 0;
+var categoryTotals2 = {};
+var testTotalsByCategory2 = {};
+
+var mean2 = 0;
+var categoryMeans2 = {};
+var testMeansByCategory2 = {};
+
+var stdDev2 = 0;
+var categoryStdDevs2 = {};
+var testStdDevsByCategory2 = {};
+
+var stdErr2 = 0;
+var categoryStdErrs2 = {};
+var testStdErrsByCategory2 = {};
+
+function initialize()
+{
+    itemTotals1 = {total: []};
+
+    for (var i = 0; i < categories.length; i++) {
+        var category = categories[i];
+        itemTotals1[category] = [];
+        categoryTotals1[category] = 0;
+        testTotalsByCategory1[category] = {};
+        categoryMeans1[category] = 0;
+        testMeansByCategory1[category] = {};
+        categoryStdDevs1[category] = 0;
+        testStdDevsByCategory1[category] = {};
+        categoryStdErrs1[category] = 0;
+        testStdErrsByCategory1[category] = {};
+    }
+
+    for (var i = 0; i < tests.length; i++) {
+        var test = tests[i];
+        itemTotals1[test] = [];
+        var category = test.replace(/-.*/, "");
+        testTotalsByCategory1[category][test] = 0;
+        testMeansByCategory1[category][test] = 0;
+        testStdDevsByCategory1[category][test] = 0;
+        testStdErrsByCategory1[category][test] = 0;
+    }
+
+    for (var i = 0; i < count1; i++) {
+        itemTotals1["total"][i] = 0;
+        for (var category in categoryTotals1) {
+            itemTotals1[category][i] = 0;
+            for (var test in testTotalsByCategory1[category]) {
+                itemTotals1[test][i] = 0;
+            }
+        }
+    }
+
+    itemTotals2 = {total: []};
+
+    for (var i = 0; i < categories.length; i++) {
+        var category = categories[i];
+        itemTotals2[category] = [];
+        categoryTotals2[category] = 0;
+        testTotalsByCategory2[category] = {};
+        categoryMeans2[category] = 0;
+        testMeansByCategory2[category] = {};
+        categoryStdDevs2[category] = 0;
+        testStdDevsByCategory2[category] = {};
+        categoryStdErrs2[category] = 0;
+        testStdErrsByCategory2[category] = {};
+    }
+
+    for (var i = 0; i < tests.length; i++) {
+        var test = tests[i];
+        itemTotals2[test] = [];
+        var category = test.replace(/-.*/, "");
+        testTotalsByCategory2[category][test] = 0;
+        testMeansByCategory2[category][test] = 0;
+        testStdDevsByCategory2[category][test] = 0;
+        testStdErrsByCategory2[category][test] = 0;
+    }
+
+    for (var i = 0; i < count2; i++) {
+        itemTotals2["total"][i] = 0;
+        for (var category in categoryTotals2) {
+            itemTotals2[category][i] = 0;
+            for (var test in testTotalsByCategory2[category]) {
+                itemTotals2[test][i] = 0;
+            }
+        }
+    }
+
+}
+
+function computeItemTotals(output, itemTotals)
+{
+    for (var i = 0; i < output.length; i++) {
+        var result = output[i];
+        for (var test in result) {
+            var time = result[test];
+            var category = test.replace(/-.*/, "");
+            itemTotals["total"][i] += time;
+            itemTotals[category][i] += time;
+            itemTotals[test][i] += time;
+        }
+    }
+}
+
+function computeTotals(output, categoryTotals, testTotalsByCategory)
+{
+    var total = 0;
+
+    for (var i = 0; i < output.length; i++) {
+        var result = output[i];
+        for (var test in result) {
+            var time = result[test];
+            var category = test.replace(/-.*/, "");
+            total += time;
+            categoryTotals[category] += time;
+            testTotalsByCategory[category][test] += time;
+        }
+    }
+
+    return total;
+}
+
+function computeMeans(count, total, categoryTotals, categoryMeans, testTotalsByCategory, testMeansByCategory)
+{
+    var mean = total / count;
+    for (var category in categoryTotals) {
+        categoryMeans[category] = categoryTotals[category] / count;
+        for (var test in testTotalsByCategory[category]) {
+            testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count;
+        }
+    }
+    return mean;
+}
+
+function standardDeviation(mean, items)
+{
+    var deltaSquaredSum = 0;
+    for (var i = 0; i < items.length; i++) {
+        var delta = items[i] - mean;
+        deltaSquaredSum += delta * delta;
+    }
+    variance = deltaSquaredSum / (items.length - 1);
+    return Math.sqrt(variance);
+}
+
+function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, testStdDevsByCategory, testMeansByCategory)
+{
+    var stdDev = standardDeviation(mean, itemTotals["total"]);
+    for (var category in categoryStdDevs) {
+        categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]);
+    }
+    for (var category in categoryStdDevs) {
+        for (var test in testStdDevsByCategory[category]) {
+            testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]);
+        }
+    }
+    return stdDev;
+}
+
+function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, testStdErrsByCategory, testStdDevsByCategory)
+{
+    var sqrtCount = Math.sqrt(count);
+    
+    var stdErr = stdDev / sqrtCount;
+    for (var category in categoryStdErrs) {
+        categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
+    }
+    for (var category in categoryStdDevs) {
+        for (var test in testStdErrsByCategory[category]) {
+            testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount;
+        }
+    }
+
+    return stdErr;
+}
+
+var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
+var tMax = tDistribution.length;
+var tLimit = 1.96;
+
+function tDist(n)
+{
+    if (n > tMax)
+        return tLimit;
+    return tDistribution[n];
+}
+
+
+function formatMean(meanWidth, mean, stdErr, count)
+{
+    var meanString = mean.toFixed(1).toString();
+    while (meanString.length < meanWidth) {
+        meanString = " " + meanString;
+    }
+
+    var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% ";
+
+    return meanString + "ms " + error;
+}
+
+function computeLabelWidth()
+{
+    var width = "Total".length;
+    for (var category in categoryMeans1) {
+        if (category.length + 2 > width)
+            width = category.length + 2;
+    }
+    for (var i = 0; i < tests.length; i++) {
+        var shortName = tests[i].replace(/^[^-]*-/, "");
+        if (shortName.length + 4 > width)
+            width = shortName.length + 4;
+    }
+
+    return width;
+}
+
+function computeMeanWidth(mean, categoryMeans, testMeansByCategory)
+{
+    var width = mean.toFixed(1).toString().length;
+    for (var category in categoryMeans) {
+        var candidate = categoryMeans[category].toFixed(1).toString().length;
+        if (candidate > width)
+            width = candidate;
+        for (var test in testMeansByCategory[category]) {
+            var candidate = testMeansByCategory[category][test].toFixed(1).toString().length;
+            if (candidate > width)
+                width = candidate;
+        }
+    }
+
+    return width;
+}
+
+function pad(str, n)
+{
+    while (str.length < n) {
+        str += " ";
+    }
+    return str;
+}
+
+function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2)
+{
+    result = pad("", indent);    
+    result += label + ": ";
+    result = pad(result, labelWidth + 2);
+
+    var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr1 * stdErr2))); 
+    var df = count1 + count2 - 2;
+
+    var statisticallySignificant = (Math.abs(t) > tDist(df+1));
+    var diff = mean2 - mean1;
+    var percentage = 100 * diff / mean1;
+    var isFaster = diff < 0;
+    var probablySame = (percentage < 0.1) && !statisticallySignificant;
+
+    var diffSummary;
+    var diffDetail;
+
+    if (probablySame) {
+        diffSummary = "-";
+        diffDetail = "";
+    } else if (!statisticallySignificant) {
+        diffSummary = "??";
+        diffDetail =  "    not conclusive: might be " + Math.abs(percentage).toFixed(1).toString() + "% ";
+        diffDetail += (isFaster ? "faster" : "*slower*");
+    } else {
+        diffSummary = Math.abs(percentage).toFixed(1).toString() + "% " + (isFaster ? "faster" : "*slower*");
+        diffDetail = "    significant"; 
+    }
+
+    return result + pad(diffSummary, 16) + formatMean(meanWidth1, mean1, stdErr1, count1) + "  " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDetail;
+}
+
+function printOutput()
+{
+    var labelWidth = computeLabelWidth();
+    var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCategory1);
+    var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCategory2);
+
+    print("\n");
+    var header = "TEST";
+    while (header.length < labelWidth)
+        header += " ";
+    header += "  COMPARISON            FROM                 TO             DETAILS";
+    print(header);
+    print("");
+    print("=============================================================================");
+    print("");
+    print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2));
+    print("");
+    print("=============================================================================");
+
+    for (var category in categoryMeans1) {
+        print("");
+        print(resultLine(labelWidth, 2, category,
+                         meanWidth1, categoryMeans1[category], categoryStdErrs1[category],
+                         meanWidth2, categoryMeans2[category], categoryStdErrs2[category]));
+        for (var test in testMeansByCategory1[category]) {
+            var shortName = test.replace(/^[^-]*-/, "");
+            print(resultLine(labelWidth, 4, shortName, 
+                             meanWidth1, testMeansByCategory1[category][test], testStdErrsByCategory1[category][test],
+                             meanWidth2, testMeansByCategory2[category][test], testStdErrsByCategory2[category][test]));
+        }
+    }
+}
+
+initialize();
+
+computeItemTotals(output1, itemTotals1);
+computeItemTotals(output2, itemTotals2);
+
+total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1);
+total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2);
+
+mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTotalsByCategory1, testMeansByCategory1);
+mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTotalsByCategory2, testMeansByCategory2);
+
+stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans1, testStdDevsByCategory1, testMeansByCategory1);
+stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans2, testStdDevsByCategory2, testMeansByCategory2);
+
+stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDevs1, testStdErrsByCategory1, testStdDevsByCategory1);
+stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDevs2, testStdErrsByCategory2, testStdDevsByCategory2);
+
+printOutput();

diff --git a/SunSpider/sunspider-compare-results b/SunSpider/sunspider-compare-results
new file mode 100755 (executable)
index 0000000..c6fa337
--- /dev/null
+++ b/SunSpider/sunspider-compare-results
@@ -0,0 +1,87 @@
+#!/usr/bin/perl -w
+
+# Copyright (C) 2007 Apple Inc.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+
+use strict;
+use File::Basename;
+
+my $programName = basename($0);
+my $usage = <<EOF;
+Usage: $programName [file 1] [file 2]
+EOF
+
+if (scalar @ARGV < 2) {
+    print STDERR $usage;
+    exit 1;
+}
+
+sub readResultsFile($)
+{
+    my ($filename) = @_;
+    open FILE, "<${filename}";
+    my $foundStart = 0;
+    my $foundEnd = 0;
+    my $result = "";
+    while (<FILE>) {
+        if (!$foundStart) {
+           if (/^[[][{]$/) {
+               $foundStart = 1;
+               $result .= $_;
+           }
+       } else {
+           if (/^[}][]]$/) {
+               $foundEnd = 1;
+               chomp;
+               $result .= $_;
+               last;
+           } else {
+               $result .= $_;
+           }
+       } 
+    }
+    close FILE;
+
+    die "Cound not find data in file - needs to be bracketed by [{ and }]." unless ($foundStart && $foundEnd);
+
+    return $result;
+}
+
+sub dumpToFile($$)
+{
+    my ($contents, $path) = @_;
+    open FILE, ">$path";
+    print FILE $contents;
+    close FILE;
+}
+
+my $output = "var output1 = " . readResultsFile($ARGV[0]) . ";\n";
+$output .= "var output2 = " . readResultsFile($ARGV[1]) . ";\n";
+
+dumpToFile($output, "tmp/sunspider-comparison-data.js");
+
+my $jsShellPath = "/Users/mjs/Work/symroots/Release/testkjs";
+
+system("$jsShellPath", "-f", "tmp/sunspider-test-prefix.js", "-f", "tmp/sunspider-comparison-data.js", "-f", "resources/sunspider-compare-results.js");
+
+