Perf dashboard should automatically select ranges for A/B testing
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 9 Apr 2015 22:57:30 +0000 (22:57 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 9 Apr 2015 22:57:30 +0000 (22:57 +0000)
https://bugs.webkit.org/show_bug.cgi?id=143580

Reviewed by Chris Dumez.

Added a new statistics option for picking a A/B test range selection strategy.
The selected ranges are shown in the graph using the same UI to show analysis tasks.

* public/v2/app.js:
(App.DashboardPaneProxyForPicker._platformOrMetricIdChanged): Updated the query parameters for
charts page used by the dashboard since we've added a new parameter at the end.
(App.Pane.ranges): Added. Merges ranges created for analysis tasks and A/B testing.
(App.Pane.updateStatisticsTools): Clone and set the test range selection strategies.
(App.Pane._cloneStrategy): Copy isSegmentation.
(App.Pane._updateMovingAverageAndEnvelope): Set testRangeCandidates.
(App.Pane._movingAverageOrEnvelopeStrategyDidChange): Update the charts when a new text range
selection strategy is picked by the user.
(App.Pane._computeMovingAverageAndOutliers): Compute the test ranges using the chosen strategy.
Avoid going through isAnomalyArray when no anomaly detection strategy is enabled. Also changed
the return value from the moving average time series to a dictionary that contains the moving
average time series, a dictionary of anomalies, and an array of test ranges.
(App.ChartsController._parsePaneList): Parse the test range selection strategy configuration.
(App.ChartsController._serializePaneList): Ditto for serialization.
(App.ChartsController._scheduleQueryStringUpdate): Update the URL hash when the user picks a new
test range selection strategy.

* public/v2/chart-pane.css: Fixed a typo as well as added a CSS rule for test ranges markers.

* public/v2/index.html: Added UI for selecting a test range selection strategy.

* public/v2/interactive-chart.js:
(App.InteractiveChartComponent._rangesChanged): Pass down "status" to be used as a class name.

* public/v2/js/statistics.js:
(Statistics.MovingAverageStrategies): Added isSegmentation to segmentation strategies.
(Statistics.TestRangeSelectionStrategies): Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@182611 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Websites/perf.webkit.org/ChangeLog
Websites/perf.webkit.org/public/v2/app.js
Websites/perf.webkit.org/public/v2/chart-pane.css
Websites/perf.webkit.org/public/v2/index.html
Websites/perf.webkit.org/public/v2/interactive-chart.js
Websites/perf.webkit.org/public/v2/js/statistics.js

index eef53c9..2d39a7c 100644 (file)
@@ -1,3 +1,42 @@
+2015-04-09  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Perf dashboard should automatically select ranges for A/B testing
+        https://bugs.webkit.org/show_bug.cgi?id=143580
+
+        Reviewed by Chris Dumez.
+
+        Added a new statistics option for picking a A/B test range selection strategy.
+        The selected ranges are shown in the graph using the same UI to show analysis tasks.
+
+        * public/v2/app.js:
+        (App.DashboardPaneProxyForPicker._platformOrMetricIdChanged): Updated the query parameters for
+        charts page used by the dashboard since we've added a new parameter at the end.
+        (App.Pane.ranges): Added. Merges ranges created for analysis tasks and A/B testing.
+        (App.Pane.updateStatisticsTools): Clone and set the test range selection strategies.
+        (App.Pane._cloneStrategy): Copy isSegmentation.
+        (App.Pane._updateMovingAverageAndEnvelope): Set testRangeCandidates.
+        (App.Pane._movingAverageOrEnvelopeStrategyDidChange): Update the charts when a new text range
+        selection strategy is picked by the user.
+        (App.Pane._computeMovingAverageAndOutliers): Compute the test ranges using the chosen strategy.
+        Avoid going through isAnomalyArray when no anomaly detection strategy is enabled. Also changed
+        the return value from the moving average time series to a dictionary that contains the moving
+        average time series, a dictionary of anomalies, and an array of test ranges.
+        (App.ChartsController._parsePaneList): Parse the test range selection strategy configuration.
+        (App.ChartsController._serializePaneList): Ditto for serialization.
+        (App.ChartsController._scheduleQueryStringUpdate): Update the URL hash when the user picks a new
+        test range selection strategy.
+
+        * public/v2/chart-pane.css: Fixed a typo as well as added a CSS rule for test ranges markers.
+
+        * public/v2/index.html: Added UI for selecting a test range selection strategy.
+
+        * public/v2/interactive-chart.js:
+        (App.InteractiveChartComponent._rangesChanged): Pass down "status" to be used as a class name.
+
+        * public/v2/js/statistics.js:
+        (Statistics.MovingAverageStrategies): Added isSegmentation to segmentation strategies.
+        (Statistics.TestRangeSelectionStrategies): Added.
+
 2015-04-08  Ryosuke Niwa  <rniwa@webkit.org>
 
         The results of A/B testing should state statistical significance
index 392bb0d..5ba08e6 100755 (executable)
@@ -52,7 +52,7 @@ App.DashboardPaneProxyForPicker = Ember.ObjectProxy.extend({
             .then(function (platforms) { self.set('pickerData', platforms); });
     }.observes('platformId', 'metricId').on('init'),
     paneList: function () {
-        return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, false]]);
+        return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, null, null, null]]);
     }.property('platformId', 'metricId'),
 });
 
@@ -425,6 +425,10 @@ App.Pane = Ember.Object.extend({
                 self.set('analyticRanges', tasks.filter(function (task) { return task.get('startRun') && task.get('endRun'); }));
             });
     },
+    ranges: function ()
+    {
+        return this.getWithDefault('analyticRanges', []).concat(this.getWithDefault('testRangeCandidates', []));
+    }.property('analyticRanges', 'testRangeCandidates'),
     _isValidId: function (id)
     {
         if (typeof(id) == "number")
@@ -500,6 +504,10 @@ App.Pane = Ember.Object.extend({
         this.set('envelopingStrategies', [{label: 'None'}].concat(envelopingStrategies));
         this.set('chosenEnvelopingStrategy', this._configureStrategy(envelopingStrategies, this.get('envelopingConfig')));
 
+        var testRangeSelectionStrategies = Statistics.TestRangeSelectionStrategies.map(this._cloneStrategy.bind(this));
+        this.set('testRangeSelectionStrategies', [{label: 'None'}].concat(testRangeSelectionStrategies));
+        this.set('chosenTestRangeSelectionStrategy', this._configureStrategy(testRangeSelectionStrategies, this.get('testRangeSelectionConfig')));
+
         var anomalyDetectionStrategies = Statistics.AnomalyDetectionStrategy.map(this._cloneStrategy.bind(this));
         this.set('anomalyDetectionStrategies', anomalyDetectionStrategies);
     }.on('init'),
@@ -509,6 +517,7 @@ App.Pane = Ember.Object.extend({
         return Ember.Object.create({
             id: strategy.id,
             label: strategy.label,
+            isSegmentation: strategy.isSegmentation,
             description: strategy.description,
             parameterList: parameterList,
             execute: strategy.execute,
@@ -542,11 +551,25 @@ App.Pane = Ember.Object.extend({
 
         var envelopingStrategy = this.get('chosenEnvelopingStrategy');
         this._updateStrategyConfigIfNeeded(envelopingStrategy, 'envelopingConfig');
-        
+
+        var testRangeSelectionStrategy = this.get('chosenTestRangeSelectionStrategy');
+        this._updateStrategyConfigIfNeeded(testRangeSelectionStrategy, 'testRangeSelectionConfig');
+
         var anomalyDetectionStrategies = this.get('anomalyDetectionStrategies').filterBy('enabled');
-        var anomalies = {};
-        chartData.movingAverage = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies);
-        this.set('highlightedItems', anomalies);
+        var result = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies);
+        if (!result)
+            return;
+
+        chartData.movingAverage = result.movingAverage;
+        this.set('highlightedItems', result.anomalies);
+        var currentTimeSeriesData = chartData.current.series();
+        this.set('testRangeCandidates', result.testRangeCandidates.map(function (range) {
+            return Ember.Object.create({
+                startRun: currentTimeSeriesData[range[0]].measurement.id(),
+                endRun: currentTimeSeriesData[range[1]].measurement.id(),
+                status: 'testingRange',
+            });
+        }));
     },
     _movingAverageOrEnvelopeStrategyDidChange: function () {
         var chartData = this.get('chartData');
@@ -555,8 +578,9 @@ App.Pane = Ember.Object.extend({
         this._setNewChartData(chartData);
     }.observes('chosenMovingAverageStrategy', 'chosenMovingAverageStrategy.parameterList.@each.value',
         'chosenEnvelopingStrategy', 'chosenEnvelopingStrategy.parameterList.@each.value',
+        'chosenTestRangeSelectionStrategy', 'chosenTestRangeSelectionStrategy.parameterList.@each.value',
         'anomalyDetectionStrategies.@each.enabled'),
-    _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies)
+    _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies)
     {
         var currentTimeSeriesData = chartData.current.series();
         var movingAverageIsSetByUser = movingAverageStrategy && movingAverageStrategy.execute;
@@ -565,6 +589,10 @@ App.Pane = Ember.Object.extend({
         if (!movingAverageValues)
             return null;
 
+        var testRangeCandidates = [];
+        if (movingAverageStrategy && movingAverageStrategy.isSegmentation && testRangeSelectionStrategy && testRangeSelectionStrategy.execute)
+            testRangeCandidates = this._executeStrategy(testRangeSelectionStrategy, currentTimeSeriesData, [movingAverageValues]);
+
         var envelopeIsSetByUser = envelopingStrategy && envelopingStrategy.execute;
         var envelopeDelta = this._executeStrategy(envelopeIsSetByUser ? envelopingStrategy : Statistics.EnvelopingStrategies[0],
             currentTimeSeriesData, [movingAverageValues]);
@@ -578,22 +606,26 @@ App.Pane = Ember.Object.extend({
         if (!envelopeIsSetByUser)
             envelopeDelta = null;
 
-        var isAnomalyArray = new Array(currentTimeSeriesData.length);
-        for (var strategy of anomalyDetectionStrategies) {
-            var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
-            for (var i = 0; i < currentTimeSeriesData.length; i++)
-                isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
-        }
-        for (var i = 0; i < isAnomalyArray.length; i++) {
-            if (!isAnomalyArray[i])
-                continue;
-            anomalies[currentTimeSeriesData[i].measurement.id()] = true;
-            while (isAnomalyArray[i] && i < isAnomalyArray.length)
-                ++i;
+        var anomalies = {};
+        if (anomalyDetectionStrategies.length) {
+            var isAnomalyArray = new Array(currentTimeSeriesData.length);
+            for (var strategy of anomalyDetectionStrategies) {
+                var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
+                for (var i = 0; i < currentTimeSeriesData.length; i++)
+                    isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
+            }
+            for (var i = 0; i < isAnomalyArray.length; i++) {
+                if (!isAnomalyArray[i])
+                    continue;
+                anomalies[currentTimeSeriesData[i].measurement.id()] = true;
+                while (isAnomalyArray[i] && i < isAnomalyArray.length)
+                    ++i;
+            }
         }
 
+        var movingAverageTimeSeries = null;
         if (movingAverageIsSetByUser) {
-            return new TimeSeries(currentTimeSeriesData.map(function (point, index) {
+            movingAverageTimeSeries = new TimeSeries(currentTimeSeriesData.map(function (point, index) {
                 var value = movingAverageValues[index];
                 return {
                     measurement: point.measurement,
@@ -601,8 +633,14 @@ App.Pane = Ember.Object.extend({
                     value: value,
                     interval: envelopeDelta !== null ? [value - envelopeDelta, value + envelopeDelta] : null,
                 }
-            }));            
+            }));
         }
+
+        return {
+            movingAverage: movingAverageTimeSeries,
+            anomalies: anomalies,
+            testRangeCandidates: testRangeCandidates,
+        };
     },
     _executeStrategy: function (strategy, currentTimeSeriesData, additionalArguments)
     {
@@ -804,6 +842,7 @@ App.ChartsController = Ember.Controller.extend({
                 showFullYAxis: paneInfo[3],
                 movingAverageConfig: paneInfo[4],
                 envelopingConfig: paneInfo[5],
+                testRangeSelectionConfig: paneInfo[6],
             });
         });
     },
@@ -821,6 +860,7 @@ App.ChartsController = Ember.Controller.extend({
                 pane.get('showFullYAxis'),
                 pane.get('movingAverageConfig'),
                 pane.get('envelopingConfig'),
+                pane.get('testRangeSelectionConfig'),
             ];
         }));
     },
@@ -829,7 +869,7 @@ App.ChartsController = Ember.Controller.extend({
     {
         Ember.run.debounce(this, '_updateQueryString', 1000);
     }.observes('sharedZoom', 'panes.@each.platform', 'panes.@each.metric', 'panes.@each.selectedItem', 'panes.@each.timeRange',
-        'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig'),
+        'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig', 'panes.@each.testRangeSelectionConfig'),
 
     _updateQueryString: function ()
     {
index da28fd1..fd524a7 100755 (executable)
@@ -135,7 +135,7 @@ a.outlier-button.show g.hide-outlier-icon {
     width: 4rem;
 }
 
-.annotation-pane {
+.analysis-pane {
     right: 1.3rem;
 }
 
@@ -461,3 +461,7 @@ a.outlier-button.show g.hide-outlier-icon {
     background-color: #fc6;
     position: absolute;
 }
+
+.chart .rangeBar.testingRange {
+    background-color: #333;
+}
index 9a88d21..5bb63f1 100755 (executable)
                     {{#if chartData}}
                         {{interactive-chart
                             chartData=chartData
-                            ranges=analyticRanges
+                            ranges=ranges
                             domain=mainPlotDomain
                             interactive=true
                             chartPointRadius=2
         <div class="rangeBarsContainerInlineStyle">
             {{#each rangeBars}}
                 {{#link-to linkRoute linkId title=label}}
-                    <span class="rangeBar" {{bind-attr style=inlineStyle}}></span>
+                    <span {{bind-attr class=":rangeBar status" style=inlineStyle}}></span>
                 {{/link-to}}
             {{/each}}
         </div>
                     {{/each}}
                 </section>
             {{/if}}
+            {{#if chosenMovingAverageStrategy.isSegmentation}}
+                <section class="stat-option">
+                    <h1>A/B Test Range Selection</h1>
+                    <label>Type: {{view Ember.Select
+                        content=testRangeSelectionStrategies
+                        optionValuePath='content'
+                        optionLabelPath='content.label'
+                        selection=chosenTestRangeSelectionStrategy}}</label>
+                    {{#if chosenTestRangeSelectionStrategy.description}}
+                        <p class="description">{{chosenTestRangeSelectionStrategy.description}}</p>
+                    {{/if}}
+                </section>
+            {{/if}}
             {{#if chosenEnvelopingStrategy.execute}}
                 <section class="stat-option">
                     <h1>Anomaly Detection</h1>
                 <div class="svg-container">
                     {{interactive-chart
                         chartData=pane.chartData
-                        ranges=pane.analyticRanges
+                        ranges=pane.ranges
                         domain=overviewDomain
                         interactive=true
                         chartPointRadius=2
index 94b1175..6eaeac3 100644 (file)
@@ -653,6 +653,7 @@ App.InteractiveChartComponent = Ember.Component.extend({
                 linkRoute: linkRoute,
                 linkId: range.get('id'),
                 label: range.get('label'),
+                status: range.get('status'),
             });
         }));
 
index 69945df..afa21e0 100755 (executable)
@@ -248,6 +248,7 @@ var Statistics = new (function () {
         },
         {
             id: 4,
+            isSegmentation: true,
             label: 'Segmentation: Recursive t-test',
             description: "Recursively split values into two segments if Welch's t-test detects a statistically significant difference.",
             parameterList: [{label: "Minimum segment length", value: 20, min: 5}],
@@ -271,6 +272,7 @@ var Statistics = new (function () {
         },
         {
             id: 5,
+            isSegmentation: true,
             label: 'Segmentation: Schwarz criterion',
             description: 'Adaptive algorithm that maximizes the Schwarz criterion (BIC).',
             // Based on Detection of Multiple Change–Points in Multivariate Time Series by Marc Lavielle (July 2006).
@@ -465,6 +467,42 @@ var Statistics = new (function () {
         },
     ];
 
+    this.debuggingTestingRangeNomination = false;
+
+    this.TestRangeSelectionStrategies = [
+        {
+            id: 301,
+            label: "t-test 99% significance",
+            execute: function (values, segmentedValues) {
+                if (!values.length)
+                    return [];
+
+                var previousMean = segmentedValues[0];
+                var selectedRanges = new Array;
+                for (var i = 1; i < segmentedValues.length; i++) {
+                    var currentMean = segmentedValues[i];
+                    if (currentMean == previousMean)
+                        continue;
+                    var found = false;
+                    for (var leftEdge = i - 2, rightEdge = i + 2; leftEdge >= 0 && rightEdge <= values.length; leftEdge--, rightEdge++) {
+                        if (segmentedValues[leftEdge] != previousMean || segmentedValues[rightEdge - 1] != currentMean)
+                            break;
+                        var result = Statistics.computeWelchsT(values, leftEdge, i - leftEdge, values, i, rightEdge - i);
+                        if (result.significantlyDifferent) {
+                            selectedRanges.push([leftEdge, rightEdge - 1]);
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (!found && Statistics.debuggingTestingRangeNomination)
+                        console.log('Failed to find a testing range at', i, 'changing from', previousValue, 'to', currentValue);
+                    previousMean = currentMean;
+                }
+                return selectedRanges;
+            }
+        }
+    ];
+
     function createWesternElectricRule(windowSize, minOutlinerCount, limitFactor) {
         return function (values, movingAverages, deviation) {
             var results = new Array(values.length);
@@ -486,7 +524,6 @@ var Statistics = new (function () {
         }
         return Math.max(valuesAboveLimit, valuesBelowLimit);
     }
-    window.countValuesOnSameSide = countValuesOnSameSide;
 
     this.AnomalyDetectionStrategy = [
         // Western Electric rules: http://en.wikipedia.org/wiki/Western_Electric_rules