Reviewed by Darin.

        * resources/sunspider-analyze-results.js: Do the error range
        properly, using the t-distribution instead of 1.96 (which was
        based on the normal distribution).

        * sunspider: Print results in a way that will be friendlier to the
        soon-to-come compare mode.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@26915 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/SunSpider/ChangeLog b/SunSpider/ChangeLog
index 50f715f..de0cdb4 100644 (file)
--- a/SunSpider/ChangeLog
+++ b/SunSpider/ChangeLog
@@ -1,3 +1,14 @@
+2007-10-23  Maciej Stachowiak  <mjs@apple.com>
+
+        Reviewed by Darin.
+
+        * resources/sunspider-analyze-results.js: Do the error range
+        properly, using the t-distribution instead of 1.96 (which was
+        based on the normal distribution).
+        
+        * sunspider: Print results in a way that will be friendlier to the
+        soon-to-come compare mode.
+
 2007-10-22  Maciej Stachowiak  <mjs@apple.com>
 
         Reviewed by Review.

diff --git a/SunSpider/resources/sunspider-analyze-results.js b/SunSpider/resources/sunspider-analyze-results.js
index 9927a0b..1b5e4f2 100644 (file)
--- a/SunSpider/resources/sunspider-analyze-results.js
+++ b/SunSpider/resources/sunspider-analyze-results.js
@@ -161,14 +161,26 @@ function computeStdErrors()
 
 }
 
-function formatResult(meanWidth, mean, stdErr)
+var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
+var tMax = tDistribution.length;
+var tLimit = 1.96;
+
+function tDist(n)
+{
+    if (n > tMax)
+        return tLimit;
+    return tDistribution[n];
+}
+
+
+function formatResult(meanWidth, mean, stdErr, n)
 {
     var meanString = mean.toFixed(1).toString();
     while (meanString.length < meanWidth) {
         meanString = " " + meanString;
     }
 
-    return meanString + "ms " + "[ +/- " + (1.96 * stdErr).toFixed(2) + "ms | +/- " + ((1.96 * stdErr / mean) * 100).toFixed(2) + "% ]";
+    return meanString + "ms " + "[ +/- " + (tDist(n) * stdErr).toFixed(2) + "ms | +/- " + ((tDist(n) * stdErr / mean) * 100).toFixed(2) + "% ]";
 }
 
 function computeLabelWidth()
@@ -217,7 +229,7 @@ function resultLine(labelWidth, indent, label, meanWidth, mean, stdErr)
         result += " ";
     }
     
-    return result + formatResult(meanWidth, mean, stdErr);
+    return result + formatResult(meanWidth, mean, stdErr, count);
 }
 
 function printOutput()

diff --git a/SunSpider/sunspider b/SunSpider/sunspider
index 17e4c97..22eda60 100755 (executable)
--- a/SunSpider/sunspider
+++ b/SunSpider/sunspider
@@ -108,7 +108,7 @@ sub runTestsOnce($)
         print STDERR "shark -i -1 -q \"$jsShellPath\" $shellArgs\n";
         $output = `shark -i -1 -q "$jsShellPath" $shellArgs`;
     } else {
-        $output = `"$jsShellPath" $shellArgs`;
+        $output = `"$jsShellPath" $shellArgs | grep -v break`;
     }
     return $output;
 }
@@ -132,12 +132,15 @@ my $result;
 my $count = 0;
 my @results = ();
 my $total = 0;
+print "[";
 while ($count++ < $testRuns) {
     $result = runTestsOnce($runShark);
     chomp $result;
     push @results, $result;
-    print $result . "\n";
+    print $result;
+    print ",\n" unless ($count == $testRuns);
 }
+print "]\n";
 
 my $output = "var output = [\n" . join(",\n", @results) . "\n];\n";
 dumpToFile($output, "tmp/sunspider-results.js");