Remove unneeded data processing in MotionMark
authorjonlee@apple.com <jonlee@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 6 Jun 2018 23:46:43 +0000 (23:46 +0000)
committerjonlee@apple.com <jonlee@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 6 Jun 2018 23:46:43 +0000 (23:46 +0000)
https://bugs.webkit.org/show_bug.cgi?id=186256

Reviewed by Said Abou-Hallawa.

Remove calculation of average frame rate based on multiple samples at a given complexity.
It is not used in the calculation of the score, and is just extra work to do at the end
of a test.

* MotionMark/developer.html: Remove options to show average data and regression based on
that data.
* MotionMark/resources/debug-runner/graph.js: Remove data. Update the time graph that
shows the relationship between the break point in the regression against the complexity
graph.
* MotionMark/resources/debug-runner/motionmark.css: Make the confidence intervals easier
to see.
* MotionMark/resources/runner/motionmark.js: Simplify score calculation now that we are
only calculating regressions on the raw data. We were calculating it based on the averages,
but that was just for analysis, and not used in the actual score. Get rid of the
|complexitySamples| local variable.
* MotionMark/resources/strings.js: Remove unneeded string constant.
* MotionMark/tests/resources/main.js:
(processSamples): Refactor so that more of it can be shared between Controller and
RampController. It also avoids processing the raw samples (controllerSamples) twice for
the ramp controller.
(_processComplexitySamples): Remove logic to calculate averages based on complexity.
(_processMarks): Split this out from the old processSamples.
(_processControllerSamples): Split this out from the old processSamples.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@232561 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog
PerformanceTests/MotionMark/developer.html
PerformanceTests/MotionMark/resources/debug-runner/graph.js
PerformanceTests/MotionMark/resources/debug-runner/motionmark.css
PerformanceTests/MotionMark/resources/runner/motionmark.js
PerformanceTests/MotionMark/resources/strings.js
PerformanceTests/MotionMark/tests/resources/main.js

index ed7ba54..32b7867 100644 (file)
@@ -1,5 +1,36 @@
 2018-06-06  Jon Lee  <jonlee@apple.com>
 
+        Remove unneeded data processing in MotionMark
+        https://bugs.webkit.org/show_bug.cgi?id=186256
+
+        Reviewed by Said Abou-Hallawa.
+
+        Remove calculation of average frame rate based on multiple samples at a given complexity.
+        It is not used in the calculation of the score, and is just extra work to do at the end
+        of a test.
+
+        * MotionMark/developer.html: Remove options to show average data and regression based on
+        that data.
+        * MotionMark/resources/debug-runner/graph.js: Remove data. Update the time graph that
+        shows the relationship between the break point in the regression against the complexity
+        graph.
+        * MotionMark/resources/debug-runner/motionmark.css: Make the confidence intervals easier
+        to see.
+        * MotionMark/resources/runner/motionmark.js: Simplify score calculation now that we are
+        only calculating regressions on the raw data. We were calculating it based on the averages,
+        but that was just for analysis, and not used in the actual score. Get rid of the
+        |complexitySamples| local variable.
+        * MotionMark/resources/strings.js: Remove unneeded string constant.
+        * MotionMark/tests/resources/main.js:
+        (processSamples): Refactor so that more of it can be shared between Controller and
+        RampController. It also avoids processing the raw samples (controllerSamples) twice for
+        the ramp controller.
+        (_processComplexitySamples): Remove logic to calculate averages based on complexity.
+        (_processMarks): Split this out from the old processSamples.
+        (_processControllerSamples): Split this out from the old processSamples.
+
+2018-06-06  Jon Lee  <jonlee@apple.com>
+
         Strip out more characters when creating permalinks
         https://bugs.webkit.org/show_bug.cgi?id=186259
 
index eb3951d..29b9144 100644 (file)
                     <form name="complexity-graph-options">
                         <ul class="series">
                             <li><label><input type="checkbox" name="series-raw" checked> Series raw</label></li>
-                            <li><label><input type="checkbox" name="series-average"> Series average</label></li>
                         </ul>
                         <ul>
                             <li><label><input type="checkbox" name="regression-time-score"> Controller score</label></li>
                             <li><label><input type="checkbox" name="bootstrap-score" checked> Bootstrap score and histogram</label></li>
                             <li><label><input type="checkbox" name="complexity-regression-aggregate-raw" checked> Regression, series raw</label><span id="complexity-regression-aggregate-raw"></span></li>
-                            <li><label><input type="checkbox" name="complexity-regression-aggregate-average"> Regression, series average</label><span id="complexity-regression-aggregate-average"></span></li>
                         </ul>
                     </form>
                 </nav>
index 9a51b7e..3073134 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -37,7 +37,7 @@ Utilities.extendObject(window.benchmarkController, {
 
         // Convert from compact JSON output to propertied data
         var samplesWithProperties = {};
-        [Strings.json.controller, Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) {
+        [Strings.json.controller, Strings.json.complexity].forEach(function(seriesName) {
             var series = testData[Strings.json.samples][seriesName];
             samplesWithProperties[seriesName] = series.toArray();
         })
@@ -92,7 +92,7 @@ Utilities.extendObject(window.benchmarkController, {
         svg.append("circle")
             .attr("cx", xScale(data.segment1[1][0]))
             .attr("cy", yScale(data.segment1[1][1]))
-            .attr("r", 5);
+            .attr("r", 3);
         this._addRegressionLine(svg, xScale, yScale, data.segment1, data.stdev);
         this._addRegressionLine(svg, xScale, yScale, data.segment2, data.stdev);
     },
@@ -158,7 +158,6 @@ Utilities.extendObject(window.benchmarkController, {
 
         // regression
         this._addRegression(result[Strings.json.complexity], svg.append("g").attr("class", "regression raw"), xScale, yScale);
-        this._addRegression(result[Strings.json.complexityAverage], svg.append("g").attr("class", "regression average"), xScale, yScale);
 
         var bootstrapResult = result[Strings.json.complexity][Strings.json.bootstrap];
         if (bootstrapResult) {
@@ -203,21 +202,6 @@ Utilities.extendObject(window.benchmarkController, {
             .attr("y1", function(d) { return yScale(d.frameLength) + 3; })
             .attr("y2", function(d) { return yScale(d.frameLength) - 3; });
 
-        group = svg.append("g")
-            .attr("class", "series average")
-            .selectAll("circle")
-                .data(data[Strings.json.complexityAverage])
-                .enter();
-        group.append("circle")
-            .attr("cx", function(d) { return xScale(d.complexity); })
-            .attr("cy", function(d) { return yScale(d.frameLength); })
-            .attr("r", 3)
-        group.append("line")
-            .attr("x1", function(d) { return xScale(d.complexity); })
-            .attr("x2", function(d) { return xScale(d.complexity); })
-            .attr("y1", function(d) { return yScale(d.frameLength - d.stdev); })
-            .attr("y2", function(d) { return yScale(d.frameLength + d.stdev); });
-
         // Cursor
         var cursorGroup = svg.append("g").attr("class", "cursor hidden");
         cursorGroup.append("line")
@@ -466,6 +450,17 @@ Utilities.extendObject(window.benchmarkController, {
                 }
                 // inflection point
                 regressionGroup.append("circle")
+                    .attr("cx", x(regression.segment2[0][0]))
+                    .attr("cy", yRight(regression.segment2[0][1]))
+                    .attr("r", 3);
+                regressionGroup.append("line")
+                    .attr("class", "association")
+                    .attr("stroke-dasharray", "5, 3")
+                    .attr("x1", x(regression.segment2[0][0]))
+                    .attr("x2", x(regression.segment2[0][0]))
+                    .attr("y1", yRight(regression.segment2[0][1]))
+                    .attr("y2", yLeft(regression.complexity));
+                regressionGroup.append("circle")
                     .attr("cx", x(regression.segment1[1][0]))
                     .attr("cy", yLeft(regression.complexity))
                     .attr("r", 5);
@@ -568,11 +563,9 @@ Utilities.extendObject(window.benchmarkController, {
     onComplexityGraphOptionsChanged: function() {
         var form = document.forms["complexity-graph-options"].elements;
         benchmarkController._showOrHideNodes(form["series-raw"].checked, "#complexity-graph .series.raw");
-        benchmarkController._showOrHideNodes(form["series-average"].checked, "#complexity-graph .series.average");
         benchmarkController._showOrHideNodes(form["regression-time-score"].checked, "#complexity-graph .mean.complexity");
         benchmarkController._showOrHideNodes(form["bootstrap-score"].checked, "#complexity-graph .bootstrap");
         benchmarkController._showOrHideNodes(form["complexity-regression-aggregate-raw"].checked, "#complexity-graph .regression.raw");
-        benchmarkController._showOrHideNodes(form["complexity-regression-aggregate-average"].checked, "#complexity-graph .regression.average");
     },
 
     onTimeGraphOptionsChanged: function() {
@@ -616,11 +609,7 @@ Utilities.extendObject(window.benchmarkController, {
             mean = mean.join("");
         } else {
             var complexityRegression = testResult[Strings.json.complexity];
-            var complexityAverageRegression = testResult[Strings.json.complexityAverage];
-
             document.getElementById("complexity-regression-aggregate-raw").textContent = complexityRegression.complexity.toFixed(2) + ", ±" + complexityRegression.stdev.toFixed(2) + "ms";
-            document.getElementById("complexity-regression-aggregate-average").textContent = complexityAverageRegression.complexity.toFixed(2) + ", ±" + complexityAverageRegression.stdev.toFixed(2) + "ms";
-
             var bootstrap = complexityRegression[Strings.json.bootstrap];
             if (bootstrap) {
                 score = bootstrap.median.toFixed(2);
index 80eb0d5..ad196db 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -704,6 +704,10 @@ body.showing-test-container.tiles-classic {
     stroke-width: 2px;
 }
 
+#regressions line.association {
+    stroke-width: 1px;
+}
+
 #regressions circle {
     fill: rgba(200, 200, 200, .8);
 }
@@ -755,7 +759,7 @@ body.showing-test-container.tiles-classic {
 }
 
 #complexity-graph .regression polygon {
-    fill: rgba(253, 253, 253, .05);
+    fill: rgba(253, 253, 253, .2);
 }
 
 #complexity-graph .raw.series line {
@@ -771,15 +775,6 @@ body.showing-test-container.tiles-classic {
     stroke: rgba(30, 96%, 86%, .05);
 }
 
-#complexity-graph .average.series circle {
-    fill: hsl(170, 96%, 56%);
-}
-
-#complexity-graph .average.series line {
-    stroke: hsla(170, 96%, 56%, .2);
-    stroke-width: 2px;
-}
-
 #complexity-graph .bootstrap .bar {
     fill: hsla(260, 56%, 66%, .4);
 }
@@ -794,5 +789,5 @@ body.showing-test-container.tiles-classic {
 }
 
 #complexity-graph .bootstrap .median polygon {
-    fill: hsla(300, 56%, 66%, .05);
+    fill: hsla(240, 56%, 66%, .4);
 }
index 40e6a21..1d16d8f 100644 (file)
@@ -92,8 +92,6 @@
         data[Strings.json.result] = result;
         var samples = data[Strings.json.samples];
 
-        var desiredFrameLength = 1000/60;
-
         function findRegression(series, profile) {
             var minIndex = Math.round(.025 * series.length);
             var maxIndex = Math.round(.975 * (series.length - 1));
 
             var complexityIndex = series.fieldMap[Strings.json.complexity];
             var frameLengthIndex = series.fieldMap[Strings.json.frameLength];
-            var regressionOptions = { desiredFrameLength: desiredFrameLength };
+            var regressionOptions = { desiredFrameLength: 1000/60 };
             if (profile)
                 regressionOptions.preferredProfile = profile;
             return {
             };
         }
 
-        var complexitySamples;
         // Convert these samples into SampleData objects if needed
-        [Strings.json.complexity, Strings.json.complexityAverage, Strings.json.controller].forEach(function(seriesName) {
+        [Strings.json.complexity, Strings.json.controller].forEach(function(seriesName) {
             var series = samples[seriesName];
             if (series && !(series instanceof SampleData))
                 samples[seriesName] = new SampleData(series.fieldMap, series.data);
             }
         }
 
-        [Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) {
-            if (!(seriesName in samples))
-                return;
-
-            var regression = {};
-            result[seriesName] = regression;
-            var regressionResult = findRegression(samples[seriesName], predominantProfile);
-            if (seriesName == Strings.json.complexity)
-                complexitySamples = regressionResult.samples;
-            var calculation = regressionResult.regression;
-            regression[Strings.json.regressions.segment1] = [
-                [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
-                [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
-            ];
-            regression[Strings.json.regressions.segment2] = [
-                [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
-                [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
-            ];
-            regression[Strings.json.complexity] = calculation.complexity;
-            regression[Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[seriesName].length);
-        });
+        var regressionResult = findRegression(samples[Strings.json.complexity], predominantProfile);
+        var calculation = regressionResult.regression;
+        result[Strings.json.complexity] = {};
+        result[Strings.json.complexity][Strings.json.regressions.segment1] = [
+            [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
+            [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
+        ];
+        result[Strings.json.complexity][Strings.json.regressions.segment2] = [
+            [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
+            [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
+        ];
+        result[Strings.json.complexity][Strings.json.complexity] = calculation.complexity;
+        result[Strings.json.complexity][Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[Strings.json.complexity].length);
 
         if (isRampController) {
             var timeComplexity = new Experiment;
             experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage();
 
             const bootstrapIterations = 2500;
-            var bootstrapResult = Regression.bootstrap(complexitySamples.data, bootstrapIterations, function(resampleData) {
-                var complexityIndex = complexitySamples.fieldMap[Strings.json.complexity];
+            var bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) {
+                var complexityIndex = regressionResult.samples.fieldMap[Strings.json.complexity];
                 resampleData.sort(function(a, b) {
                     return a[complexityIndex] - b[complexityIndex];
                 });
 
-                var resample = new SampleData(complexitySamples.fieldMap, resampleData);
-                var regressionResult = findRegression(resample, predominantProfile);
-                return regressionResult.regression.complexity;
+                var resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
+                var bootstrapRegressionResult = findRegression(resample, predominantProfile);
+                return bootstrapRegressionResult.regression.complexity;
             }, .8);
 
             result[Strings.json.complexity][Strings.json.bootstrap] = bootstrapResult;
index 1ba70a4..55a47e7 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -37,7 +37,6 @@ var Strings = {
         controller: "controller",
         time: "time",
         complexity: "complexity",
-        complexityAverage: "complexityAverage",
         frameLength: "frameLength",
         smoothedFrameLength: "smoothedFrameLength",
 
index 093eb58..bfeefdf 100644 (file)
@@ -171,72 +171,31 @@ Controller = Utilities.createClass(
         return this._sampler.processSamples();
     },
 
-    _processComplexitySamples: function(complexitySamples, complexityAverageSamples)
+    _processComplexitySamples: function(complexitySamples)
     {
-        complexityAverageSamples.addField(Strings.json.complexity, 0);
-        complexityAverageSamples.addField(Strings.json.frameLength, 1);
-        complexityAverageSamples.addField(Strings.json.measurements.stdev, 2);
-
         complexitySamples.sort(function(a, b) {
             return complexitySamples.getFieldInDatum(a, Strings.json.complexity) - complexitySamples.getFieldInDatum(b, Strings.json.complexity);
         });
-
-        // Samples averaged based on complexity
-        var currentComplexity = -1;
-        var experimentAtComplexity;
-        function addSample() {
-            var mean = experimentAtComplexity.mean();
-            var stdev = experimentAtComplexity.standardDeviation();
-
-            var averageSample = complexityAverageSamples.createDatum();
-            complexityAverageSamples.push(averageSample);
-            complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.complexity, currentComplexity);
-            complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.frameLength, mean);
-            complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.measurements.stdev, stdev);
-        }
-        complexitySamples.forEach(function(sample) {
-            var sampleComplexity = complexitySamples.getFieldInDatum(sample, Strings.json.complexity);
-            if (sampleComplexity != currentComplexity) {
-                if (currentComplexity > -1)
-                    addSample();
-
-                currentComplexity = sampleComplexity;
-                experimentAtComplexity = new Experiment;
-            }
-            experimentAtComplexity.sample(complexitySamples.getFieldInDatum(sample, Strings.json.frameLength));
-        });
-        // Finish off the last one
-        addSample();
     },
 
-    processSamples: function(results)
+    _processMarks: function()
     {
-        var complexityExperiment = new Experiment;
-        var smoothedFrameLengthExperiment = new Experiment;
-
-        var samples = this._sampler.samples;
-
         for (var markName in this._marks)
             this._marks[markName].time -= this._startTimestamp;
-        results[Strings.json.marks] = this._marks;
-
-        results[Strings.json.samples] = {};
-
+        return this._marks;
+    },
+    _processControllerSamples: function()
+    {
         var controllerSamples = new SampleData;
-        results[Strings.json.samples][Strings.json.controller] = controllerSamples;
-
         controllerSamples.addField(Strings.json.time, 0);
         controllerSamples.addField(Strings.json.complexity, 1);
         controllerSamples.addField(Strings.json.frameLength, 2);
         controllerSamples.addField(Strings.json.smoothedFrameLength, 3);
 
-        var complexitySamples = new SampleData(controllerSamples.fieldMap);
-        results[Strings.json.samples][Strings.json.complexity] = complexitySamples;
-
+        var samples = this._sampler.samples;
         samples[0].forEach(function(timestamp, i) {
             var sample = controllerSamples.createDatum();
             controllerSamples.push(sample);
-            complexitySamples.push(sample);
 
             // Represent time in milliseconds
             controllerSamples.setFieldInDatum(sample, Strings.json.time, timestamp - this._startTimestamp);
@@ -251,9 +210,23 @@ Controller = Utilities.createClass(
                 controllerSamples.setFieldInDatum(sample, Strings.json.smoothedFrameLength, samples[2][i]);
         }, this);
 
-        var complexityAverageSamples = new SampleData;
-        results[Strings.json.samples][Strings.json.complexityAverage] = complexityAverageSamples;
-        this._processComplexitySamples(complexitySamples, complexityAverageSamples);
+        return controllerSamples;
+    },
+
+    processSamples: function(results)
+    {
+        results[Strings.json.marks] = this._processMarks();
+
+        var controllerSamples = this._processControllerSamples();
+        var complexitySamples = new SampleData(controllerSamples.fieldMap);
+
+        results[Strings.json.samples] = {};
+        results[Strings.json.samples][Strings.json.controller] = controllerSamples;
+        results[Strings.json.samples][Strings.json.complexity] = complexitySamples;
+        controllerSamples.forEach(function (sample) {
+            complexitySamples.push(sample);
+        });
+        this._processComplexitySamples(complexitySamples);
     }
 });
 
@@ -546,16 +519,17 @@ RampController = Utilities.createSubclass(Controller,
 
     processSamples: function(results)
     {
-        Controller.prototype.processSamples.call(this, results);
-
+        results[Strings.json.marks] = this._processMarks();
         // Have samplingTimeOffset represent time 0
         var startTimestamp = this._marks[Strings.json.samplingStartTimeOffset].time;
-
         for (var markName in results[Strings.json.marks]) {
             results[Strings.json.marks][markName].time -= startTimestamp;
         }
 
-        var controllerSamples = results[Strings.json.samples][Strings.json.controller];
+        results[Strings.json.samples] = {};
+
+        var controllerSamples = this._processControllerSamples();
+        results[Strings.json.samples][Strings.json.controller] = controllerSamples;
         controllerSamples.forEach(function(timeSample) {
             controllerSamples.setFieldInDatum(timeSample, Strings.json.time, controllerSamples.getFieldInDatum(timeSample, Strings.json.time) - startTimestamp);
         });
@@ -595,9 +569,7 @@ RampController = Utilities.createSubclass(Controller,
                 complexitySamples.push(controllerSamples.at(j));
         });
 
-        var complexityAverageSamples = new SampleData;
-        results[Strings.json.samples][Strings.json.complexityAverage] = complexityAverageSamples;
-        this._processComplexitySamples(complexitySamples, complexityAverageSamples);
+        this._processComplexitySamples(complexitySamples);
     }
 });