Perf dashboard should automatically select ranges for A/B testing
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 9 Apr 2015 22:57:30 +0000 (22:57 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 9 Apr 2015 22:57:30 +0000 (22:57 +0000)
https://bugs.webkit.org/show_bug.cgi?id=143580

Reviewed by Chris Dumez.

Added a new statistics option for picking a A/B test range selection strategy.
The selected ranges are shown in the graph using the same UI to show analysis tasks.

* public/v2/app.js:
(App.DashboardPaneProxyForPicker._platformOrMetricIdChanged): Updated the query parameters for
charts page used by the dashboard since we've added a new parameter at the end.
(App.Pane.ranges): Added. Merges ranges created for analysis tasks and A/B testing.
(App.Pane.updateStatisticsTools): Clone and set the test range selection strategies.
(App.Pane._cloneStrategy): Copy isSegmentation.
(App.Pane._updateMovingAverageAndEnvelope): Set testRangeCandidates.
(App.Pane._movingAverageOrEnvelopeStrategyDidChange): Update the charts when a new text range
selection strategy is picked by the user.
(App.Pane._computeMovingAverageAndOutliers): Compute the test ranges using the chosen strategy.
Avoid going through isAnomalyArray when no anomaly detection strategy is enabled. Also changed
the return value from the moving average time series to a dictionary that contains the moving
average time series, a dictionary of anomalies, and an array of test ranges.
(App.ChartsController._parsePaneList): Parse the test range selection strategy configuration.
(App.ChartsController._serializePaneList): Ditto for serialization.
(App.ChartsController._scheduleQueryStringUpdate): Update the URL hash when the user picks a new
test range selection strategy.

* public/v2/chart-pane.css: Fixed a typo as well as added a CSS rule for test ranges markers.

* public/v2/index.html: Added UI for selecting a test range selection strategy.

* public/v2/interactive-chart.js:
(App.InteractiveChartComponent._rangesChanged): Pass down "status" to be used as a class name.

* public/v2/js/statistics.js:
(Statistics.MovingAverageStrategies): Added isSegmentation to segmentation strategies.
(Statistics.TestRangeSelectionStrategies): Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@182611 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Websites/perf.webkit.org/ChangeLog
Websites/perf.webkit.org/public/v2/app.js
Websites/perf.webkit.org/public/v2/chart-pane.css
Websites/perf.webkit.org/public/v2/index.html
Websites/perf.webkit.org/public/v2/interactive-chart.js
Websites/perf.webkit.org/public/v2/js/statistics.js

index eef53c9f6d1bf15d18b7fcb3fa04a9474cfaaf8d..2d39a7ca27b085f90fbf1be9b078844bf7ebf1e5 100644 (file)
@@ -1,3 +1,42 @@
+2015-04-09  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Perf dashboard should automatically select ranges for A/B testing
+        https://bugs.webkit.org/show_bug.cgi?id=143580
+
+        Reviewed by Chris Dumez.
+
+        Added a new statistics option for picking a A/B test range selection strategy.
+        The selected ranges are shown in the graph using the same UI to show analysis tasks.
+
+        * public/v2/app.js:
+        (App.DashboardPaneProxyForPicker._platformOrMetricIdChanged): Updated the query parameters for
+        charts page used by the dashboard since we've added a new parameter at the end.
+        (App.Pane.ranges): Added. Merges ranges created for analysis tasks and A/B testing.
+        (App.Pane.updateStatisticsTools): Clone and set the test range selection strategies.
+        (App.Pane._cloneStrategy): Copy isSegmentation.
+        (App.Pane._updateMovingAverageAndEnvelope): Set testRangeCandidates.
+        (App.Pane._movingAverageOrEnvelopeStrategyDidChange): Update the charts when a new text range
+        selection strategy is picked by the user.
+        (App.Pane._computeMovingAverageAndOutliers): Compute the test ranges using the chosen strategy.
+        Avoid going through isAnomalyArray when no anomaly detection strategy is enabled. Also changed
+        the return value from the moving average time series to a dictionary that contains the moving
+        average time series, a dictionary of anomalies, and an array of test ranges.
+        (App.ChartsController._parsePaneList): Parse the test range selection strategy configuration.
+        (App.ChartsController._serializePaneList): Ditto for serialization.
+        (App.ChartsController._scheduleQueryStringUpdate): Update the URL hash when the user picks a new
+        test range selection strategy.
+
+        * public/v2/chart-pane.css: Fixed a typo as well as added a CSS rule for test ranges markers.
+
+        * public/v2/index.html: Added UI for selecting a test range selection strategy.
+
+        * public/v2/interactive-chart.js:
+        (App.InteractiveChartComponent._rangesChanged): Pass down "status" to be used as a class name.
+
+        * public/v2/js/statistics.js:
+        (Statistics.MovingAverageStrategies): Added isSegmentation to segmentation strategies.
+        (Statistics.TestRangeSelectionStrategies): Added.
+
 2015-04-08  Ryosuke Niwa  <rniwa@webkit.org>
 
         The results of A/B testing should state statistical significance
index 392bb0d1527481090f7fcf290d2c284bbc93f9e1..5ba08e6333753c4f8c8518ff9cf74758c3b610b1 100755 (executable)
@@ -52,7 +52,7 @@ App.DashboardPaneProxyForPicker = Ember.ObjectProxy.extend({
             .then(function (platforms) { self.set('pickerData', platforms); });
     }.observes('platformId', 'metricId').on('init'),
     paneList: function () {
-        return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, false]]);
+        return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, null, null, null]]);
     }.property('platformId', 'metricId'),
 });
 
@@ -425,6 +425,10 @@ App.Pane = Ember.Object.extend({
                 self.set('analyticRanges', tasks.filter(function (task) { return task.get('startRun') && task.get('endRun'); }));
             });
     },
+    ranges: function ()
+    {
+        return this.getWithDefault('analyticRanges', []).concat(this.getWithDefault('testRangeCandidates', []));
+    }.property('analyticRanges', 'testRangeCandidates'),
     _isValidId: function (id)
     {
         if (typeof(id) == "number")
@@ -500,6 +504,10 @@ App.Pane = Ember.Object.extend({
         this.set('envelopingStrategies', [{label: 'None'}].concat(envelopingStrategies));
         this.set('chosenEnvelopingStrategy', this._configureStrategy(envelopingStrategies, this.get('envelopingConfig')));
 
+        var testRangeSelectionStrategies = Statistics.TestRangeSelectionStrategies.map(this._cloneStrategy.bind(this));
+        this.set('testRangeSelectionStrategies', [{label: 'None'}].concat(testRangeSelectionStrategies));
+        this.set('chosenTestRangeSelectionStrategy', this._configureStrategy(testRangeSelectionStrategies, this.get('testRangeSelectionConfig')));
+
         var anomalyDetectionStrategies = Statistics.AnomalyDetectionStrategy.map(this._cloneStrategy.bind(this));
         this.set('anomalyDetectionStrategies', anomalyDetectionStrategies);
     }.on('init'),
@@ -509,6 +517,7 @@ App.Pane = Ember.Object.extend({
         return Ember.Object.create({
             id: strategy.id,
             label: strategy.label,
+            isSegmentation: strategy.isSegmentation,
             description: strategy.description,
             parameterList: parameterList,
             execute: strategy.execute,
@@ -542,11 +551,25 @@ App.Pane = Ember.Object.extend({
 
         var envelopingStrategy = this.get('chosenEnvelopingStrategy');
         this._updateStrategyConfigIfNeeded(envelopingStrategy, 'envelopingConfig');
-        
+
+        var testRangeSelectionStrategy = this.get('chosenTestRangeSelectionStrategy');
+        this._updateStrategyConfigIfNeeded(testRangeSelectionStrategy, 'testRangeSelectionConfig');
+
         var anomalyDetectionStrategies = this.get('anomalyDetectionStrategies').filterBy('enabled');
-        var anomalies = {};
-        chartData.movingAverage = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies);
-        this.set('highlightedItems', anomalies);
+        var result = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies);
+        if (!result)
+            return;
+
+        chartData.movingAverage = result.movingAverage;
+        this.set('highlightedItems', result.anomalies);
+        var currentTimeSeriesData = chartData.current.series();
+        this.set('testRangeCandidates', result.testRangeCandidates.map(function (range) {
+            return Ember.Object.create({
+                startRun: currentTimeSeriesData[range[0]].measurement.id(),
+                endRun: currentTimeSeriesData[range[1]].measurement.id(),
+                status: 'testingRange',
+            });
+        }));
     },
     _movingAverageOrEnvelopeStrategyDidChange: function () {
         var chartData = this.get('chartData');
@@ -555,8 +578,9 @@ App.Pane = Ember.Object.extend({
         this._setNewChartData(chartData);
     }.observes('chosenMovingAverageStrategy', 'chosenMovingAverageStrategy.parameterList.@each.value',
         'chosenEnvelopingStrategy', 'chosenEnvelopingStrategy.parameterList.@each.value',
+        'chosenTestRangeSelectionStrategy', 'chosenTestRangeSelectionStrategy.parameterList.@each.value',
         'anomalyDetectionStrategies.@each.enabled'),
-    _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies)
+    _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies)
     {
         var currentTimeSeriesData = chartData.current.series();
         var movingAverageIsSetByUser = movingAverageStrategy && movingAverageStrategy.execute;
@@ -565,6 +589,10 @@ App.Pane = Ember.Object.extend({
         if (!movingAverageValues)
             return null;
 
+        var testRangeCandidates = [];
+        if (movingAverageStrategy && movingAverageStrategy.isSegmentation && testRangeSelectionStrategy && testRangeSelectionStrategy.execute)
+            testRangeCandidates = this._executeStrategy(testRangeSelectionStrategy, currentTimeSeriesData, [movingAverageValues]);
+
         var envelopeIsSetByUser = envelopingStrategy && envelopingStrategy.execute;
         var envelopeDelta = this._executeStrategy(envelopeIsSetByUser ? envelopingStrategy : Statistics.EnvelopingStrategies[0],
             currentTimeSeriesData, [movingAverageValues]);
@@ -578,22 +606,26 @@ App.Pane = Ember.Object.extend({
         if (!envelopeIsSetByUser)
             envelopeDelta = null;
 
-        var isAnomalyArray = new Array(currentTimeSeriesData.length);
-        for (var strategy of anomalyDetectionStrategies) {
-            var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
-            for (var i = 0; i < currentTimeSeriesData.length; i++)
-                isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
-        }
-        for (var i = 0; i < isAnomalyArray.length; i++) {
-            if (!isAnomalyArray[i])
-                continue;
-            anomalies[currentTimeSeriesData[i].measurement.id()] = true;
-            while (isAnomalyArray[i] && i < isAnomalyArray.length)
-                ++i;
+        var anomalies = {};
+        if (anomalyDetectionStrategies.length) {
+            var isAnomalyArray = new Array(currentTimeSeriesData.length);
+            for (var strategy of anomalyDetectionStrategies) {
+                var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
+                for (var i = 0; i < currentTimeSeriesData.length; i++)
+                    isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
+            }
+            for (var i = 0; i < isAnomalyArray.length; i++) {
+                if (!isAnomalyArray[i])
+                    continue;
+                anomalies[currentTimeSeriesData[i].measurement.id()] = true;
+                while (isAnomalyArray[i] && i < isAnomalyArray.length)
+                    ++i;
+            }
         }
 
+        var movingAverageTimeSeries = null;
         if (movingAverageIsSetByUser) {
-            return new TimeSeries(currentTimeSeriesData.map(function (point, index) {
+            movingAverageTimeSeries = new TimeSeries(currentTimeSeriesData.map(function (point, index) {
                 var value = movingAverageValues[index];
                 return {
                     measurement: point.measurement,
@@ -601,8 +633,14 @@ App.Pane = Ember.Object.extend({
                     value: value,
                     interval: envelopeDelta !== null ? [value - envelopeDelta, value + envelopeDelta] : null,
                 }
-            }));            
+            }));
         }
+
+        return {
+            movingAverage: movingAverageTimeSeries,
+            anomalies: anomalies,
+            testRangeCandidates: testRangeCandidates,
+        };
     },
     _executeStrategy: function (strategy, currentTimeSeriesData, additionalArguments)
     {
@@ -804,6 +842,7 @@ App.ChartsController = Ember.Controller.extend({
                 showFullYAxis: paneInfo[3],
                 movingAverageConfig: paneInfo[4],
                 envelopingConfig: paneInfo[5],
+                testRangeSelectionConfig: paneInfo[6],
             });
         });
     },
@@ -821,6 +860,7 @@ App.ChartsController = Ember.Controller.extend({
                 pane.get('showFullYAxis'),
                 pane.get('movingAverageConfig'),
                 pane.get('envelopingConfig'),
+                pane.get('testRangeSelectionConfig'),
             ];
         }));
     },
@@ -829,7 +869,7 @@ App.ChartsController = Ember.Controller.extend({
     {
         Ember.run.debounce(this, '_updateQueryString', 1000);
     }.observes('sharedZoom', 'panes.@each.platform', 'panes.@each.metric', 'panes.@each.selectedItem', 'panes.@each.timeRange',
-        'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig'),
+        'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig', 'panes.@each.testRangeSelectionConfig'),
 
     _updateQueryString: function ()
     {
index da28fd17cc705cdef0023a5af0980215ced1db25..fd524a7d0cdeb7f327bc02975aef6cb144333c58 100755 (executable)
@@ -135,7 +135,7 @@ a.outlier-button.show g.hide-outlier-icon {
     width: 4rem;
 }
 
-.annotation-pane {
+.analysis-pane {
     right: 1.3rem;
 }
 
@@ -461,3 +461,7 @@ a.outlier-button.show g.hide-outlier-icon {
     background-color: #fc6;
     position: absolute;
 }
+
+.chart .rangeBar.testingRange {
+    background-color: #333;
+}
index 9a88d2172183104ee18287f3ff5a9b688cf4b2ef..5bb63f18ee2234024b8db51dd9ab2d516e597a63 100755 (executable)
                     {{#if chartData}}
                         {{interactive-chart
                             chartData=chartData
-                            ranges=analyticRanges
+                            ranges=ranges
                             domain=mainPlotDomain
                             interactive=true
                             chartPointRadius=2
         <div class="rangeBarsContainerInlineStyle">
             {{#each rangeBars}}
                 {{#link-to linkRoute linkId title=label}}
-                    <span class="rangeBar" {{bind-attr style=inlineStyle}}></span>
+                    <span {{bind-attr class=":rangeBar status" style=inlineStyle}}></span>
                 {{/link-to}}
             {{/each}}
         </div>
                     {{/each}}
                 </section>
             {{/if}}
+            {{#if chosenMovingAverageStrategy.isSegmentation}}
+                <section class="stat-option">
+                    <h1>A/B Test Range Selection</h1>
+                    <label>Type: {{view Ember.Select
+                        content=testRangeSelectionStrategies
+                        optionValuePath='content'
+                        optionLabelPath='content.label'
+                        selection=chosenTestRangeSelectionStrategy}}</label>
+                    {{#if chosenTestRangeSelectionStrategy.description}}
+                        <p class="description">{{chosenTestRangeSelectionStrategy.description}}</p>
+                    {{/if}}
+                </section>
+            {{/if}}
             {{#if chosenEnvelopingStrategy.execute}}
                 <section class="stat-option">
                     <h1>Anomaly Detection</h1>
                 <div class="svg-container">
                     {{interactive-chart
                         chartData=pane.chartData
-                        ranges=pane.analyticRanges
+                        ranges=pane.ranges
                         domain=overviewDomain
                         interactive=true
                         chartPointRadius=2
index 94b1175f56b017bba8f7819e5783c907c87d151f..6eaeac3dca44cd76cf604f64faf8cc0ced62670e 100644 (file)
@@ -653,6 +653,7 @@ App.InteractiveChartComponent = Ember.Component.extend({
                 linkRoute: linkRoute,
                 linkId: range.get('id'),
                 label: range.get('label'),
+                status: range.get('status'),
             });
         }));
 
index 69945df8b28c941b201f32dd6ca912196814363a..afa21e004f340fcb3da19cea96486384d15eb9f5 100755 (executable)
@@ -248,6 +248,7 @@ var Statistics = new (function () {
         },
         {
             id: 4,
+            isSegmentation: true,
             label: 'Segmentation: Recursive t-test',
             description: "Recursively split values into two segments if Welch's t-test detects a statistically significant difference.",
             parameterList: [{label: "Minimum segment length", value: 20, min: 5}],
@@ -271,6 +272,7 @@ var Statistics = new (function () {
         },
         {
             id: 5,
+            isSegmentation: true,
             label: 'Segmentation: Schwarz criterion',
             description: 'Adaptive algorithm that maximizes the Schwarz criterion (BIC).',
             // Based on Detection of Multiple Change–Points in Multivariate Time Series by Marc Lavielle (July 2006).
@@ -465,6 +467,42 @@ var Statistics = new (function () {
         },
     ];
 
+    this.debuggingTestingRangeNomination = false;
+
+    this.TestRangeSelectionStrategies = [
+        {
+            id: 301,
+            label: "t-test 99% significance",
+            execute: function (values, segmentedValues) {
+                if (!values.length)
+                    return [];
+
+                var previousMean = segmentedValues[0];
+                var selectedRanges = new Array;
+                for (var i = 1; i < segmentedValues.length; i++) {
+                    var currentMean = segmentedValues[i];
+                    if (currentMean == previousMean)
+                        continue;
+                    var found = false;
+                    for (var leftEdge = i - 2, rightEdge = i + 2; leftEdge >= 0 && rightEdge <= values.length; leftEdge--, rightEdge++) {
+                        if (segmentedValues[leftEdge] != previousMean || segmentedValues[rightEdge - 1] != currentMean)
+                            break;
+                        var result = Statistics.computeWelchsT(values, leftEdge, i - leftEdge, values, i, rightEdge - i);
+                        if (result.significantlyDifferent) {
+                            selectedRanges.push([leftEdge, rightEdge - 1]);
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (!found && Statistics.debuggingTestingRangeNomination)
+                        console.log('Failed to find a testing range at', i, 'changing from', previousValue, 'to', currentValue);
+                    previousMean = currentMean;
+                }
+                return selectedRanges;
+            }
+        }
+    ];
+
     function createWesternElectricRule(windowSize, minOutlinerCount, limitFactor) {
         return function (values, movingAverages, deviation) {
             var results = new Array(values.length);
@@ -486,7 +524,6 @@ var Statistics = new (function () {
         }
         return Math.max(valuesAboveLimit, valuesBelowLimit);
     }
-    window.countValuesOnSameSide = countValuesOnSameSide;
 
     this.AnomalyDetectionStrategy = [
         // Western Electric rules: http://en.wikipedia.org/wiki/Western_Electric_rules