+2015-04-09 Ryosuke Niwa <rniwa@webkit.org>
+
+ Perf dashboard should automatically select ranges for A/B testing
+ https://bugs.webkit.org/show_bug.cgi?id=143580
+
+ Reviewed by Chris Dumez.
+
+ Added a new statistics option for picking a A/B test range selection strategy.
+ The selected ranges are shown in the graph using the same UI to show analysis tasks.
+
+ * public/v2/app.js:
+ (App.DashboardPaneProxyForPicker._platformOrMetricIdChanged): Updated the query parameters for
+ charts page used by the dashboard since we've added a new parameter at the end.
+ (App.Pane.ranges): Added. Merges ranges created for analysis tasks and A/B testing.
+ (App.Pane.updateStatisticsTools): Clone and set the test range selection strategies.
+ (App.Pane._cloneStrategy): Copy isSegmentation.
+ (App.Pane._updateMovingAverageAndEnvelope): Set testRangeCandidates.
+ (App.Pane._movingAverageOrEnvelopeStrategyDidChange): Update the charts when a new text range
+ selection strategy is picked by the user.
+ (App.Pane._computeMovingAverageAndOutliers): Compute the test ranges using the chosen strategy.
+ Avoid going through isAnomalyArray when no anomaly detection strategy is enabled. Also changed
+ the return value from the moving average time series to a dictionary that contains the moving
+ average time series, a dictionary of anomalies, and an array of test ranges.
+ (App.ChartsController._parsePaneList): Parse the test range selection strategy configuration.
+ (App.ChartsController._serializePaneList): Ditto for serialization.
+ (App.ChartsController._scheduleQueryStringUpdate): Update the URL hash when the user picks a new
+ test range selection strategy.
+
+ * public/v2/chart-pane.css: Fixed a typo as well as added a CSS rule for test ranges markers.
+
+ * public/v2/index.html: Added UI for selecting a test range selection strategy.
+
+ * public/v2/interactive-chart.js:
+ (App.InteractiveChartComponent._rangesChanged): Pass down "status" to be used as a class name.
+
+ * public/v2/js/statistics.js:
+ (Statistics.MovingAverageStrategies): Added isSegmentation to segmentation strategies.
+ (Statistics.TestRangeSelectionStrategies): Added.
+
2015-04-08 Ryosuke Niwa <rniwa@webkit.org>
The results of A/B testing should state statistical significance
.then(function (platforms) { self.set('pickerData', platforms); });
}.observes('platformId', 'metricId').on('init'),
paneList: function () {
- return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, false]]);
+ return App.encodePrettifiedJSON([[this.get('platformId'), this.get('metricId'), null, null, null, null, null]]);
}.property('platformId', 'metricId'),
});
self.set('analyticRanges', tasks.filter(function (task) { return task.get('startRun') && task.get('endRun'); }));
});
},
+ ranges: function ()
+ {
+ return this.getWithDefault('analyticRanges', []).concat(this.getWithDefault('testRangeCandidates', []));
+ }.property('analyticRanges', 'testRangeCandidates'),
_isValidId: function (id)
{
if (typeof(id) == "number")
this.set('envelopingStrategies', [{label: 'None'}].concat(envelopingStrategies));
this.set('chosenEnvelopingStrategy', this._configureStrategy(envelopingStrategies, this.get('envelopingConfig')));
+ var testRangeSelectionStrategies = Statistics.TestRangeSelectionStrategies.map(this._cloneStrategy.bind(this));
+ this.set('testRangeSelectionStrategies', [{label: 'None'}].concat(testRangeSelectionStrategies));
+ this.set('chosenTestRangeSelectionStrategy', this._configureStrategy(testRangeSelectionStrategies, this.get('testRangeSelectionConfig')));
+
var anomalyDetectionStrategies = Statistics.AnomalyDetectionStrategy.map(this._cloneStrategy.bind(this));
this.set('anomalyDetectionStrategies', anomalyDetectionStrategies);
}.on('init'),
return Ember.Object.create({
id: strategy.id,
label: strategy.label,
+ isSegmentation: strategy.isSegmentation,
description: strategy.description,
parameterList: parameterList,
execute: strategy.execute,
var envelopingStrategy = this.get('chosenEnvelopingStrategy');
this._updateStrategyConfigIfNeeded(envelopingStrategy, 'envelopingConfig');
-
+
+ var testRangeSelectionStrategy = this.get('chosenTestRangeSelectionStrategy');
+ this._updateStrategyConfigIfNeeded(testRangeSelectionStrategy, 'testRangeSelectionConfig');
+
var anomalyDetectionStrategies = this.get('anomalyDetectionStrategies').filterBy('enabled');
- var anomalies = {};
- chartData.movingAverage = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies);
- this.set('highlightedItems', anomalies);
+ var result = this._computeMovingAverageAndOutliers(chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies);
+ if (!result)
+ return;
+
+ chartData.movingAverage = result.movingAverage;
+ this.set('highlightedItems', result.anomalies);
+ var currentTimeSeriesData = chartData.current.series();
+ this.set('testRangeCandidates', result.testRangeCandidates.map(function (range) {
+ return Ember.Object.create({
+ startRun: currentTimeSeriesData[range[0]].measurement.id(),
+ endRun: currentTimeSeriesData[range[1]].measurement.id(),
+ status: 'testingRange',
+ });
+ }));
},
_movingAverageOrEnvelopeStrategyDidChange: function () {
var chartData = this.get('chartData');
this._setNewChartData(chartData);
}.observes('chosenMovingAverageStrategy', 'chosenMovingAverageStrategy.parameterList.@each.value',
'chosenEnvelopingStrategy', 'chosenEnvelopingStrategy.parameterList.@each.value',
+ 'chosenTestRangeSelectionStrategy', 'chosenTestRangeSelectionStrategy.parameterList.@each.value',
'anomalyDetectionStrategies.@each.enabled'),
- _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, anomalyDetectionStrategies, anomalies)
+ _computeMovingAverageAndOutliers: function (chartData, movingAverageStrategy, envelopingStrategy, testRangeSelectionStrategy, anomalyDetectionStrategies)
{
var currentTimeSeriesData = chartData.current.series();
var movingAverageIsSetByUser = movingAverageStrategy && movingAverageStrategy.execute;
if (!movingAverageValues)
return null;
+ var testRangeCandidates = [];
+ if (movingAverageStrategy && movingAverageStrategy.isSegmentation && testRangeSelectionStrategy && testRangeSelectionStrategy.execute)
+ testRangeCandidates = this._executeStrategy(testRangeSelectionStrategy, currentTimeSeriesData, [movingAverageValues]);
+
var envelopeIsSetByUser = envelopingStrategy && envelopingStrategy.execute;
var envelopeDelta = this._executeStrategy(envelopeIsSetByUser ? envelopingStrategy : Statistics.EnvelopingStrategies[0],
currentTimeSeriesData, [movingAverageValues]);
if (!envelopeIsSetByUser)
envelopeDelta = null;
- var isAnomalyArray = new Array(currentTimeSeriesData.length);
- for (var strategy of anomalyDetectionStrategies) {
- var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
- for (var i = 0; i < currentTimeSeriesData.length; i++)
- isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
- }
- for (var i = 0; i < isAnomalyArray.length; i++) {
- if (!isAnomalyArray[i])
- continue;
- anomalies[currentTimeSeriesData[i].measurement.id()] = true;
- while (isAnomalyArray[i] && i < isAnomalyArray.length)
- ++i;
+ var anomalies = {};
+ if (anomalyDetectionStrategies.length) {
+ var isAnomalyArray = new Array(currentTimeSeriesData.length);
+ for (var strategy of anomalyDetectionStrategies) {
+ var anomalyLengths = this._executeStrategy(strategy, currentTimeSeriesData, [movingAverageValues, envelopeDelta]);
+ for (var i = 0; i < currentTimeSeriesData.length; i++)
+ isAnomalyArray[i] = isAnomalyArray[i] || anomalyLengths[i];
+ }
+ for (var i = 0; i < isAnomalyArray.length; i++) {
+ if (!isAnomalyArray[i])
+ continue;
+ anomalies[currentTimeSeriesData[i].measurement.id()] = true;
+ while (isAnomalyArray[i] && i < isAnomalyArray.length)
+ ++i;
+ }
}
+ var movingAverageTimeSeries = null;
if (movingAverageIsSetByUser) {
- return new TimeSeries(currentTimeSeriesData.map(function (point, index) {
+ movingAverageTimeSeries = new TimeSeries(currentTimeSeriesData.map(function (point, index) {
var value = movingAverageValues[index];
return {
measurement: point.measurement,
value: value,
interval: envelopeDelta !== null ? [value - envelopeDelta, value + envelopeDelta] : null,
}
- }));
+ }));
}
+
+ return {
+ movingAverage: movingAverageTimeSeries,
+ anomalies: anomalies,
+ testRangeCandidates: testRangeCandidates,
+ };
},
_executeStrategy: function (strategy, currentTimeSeriesData, additionalArguments)
{
showFullYAxis: paneInfo[3],
movingAverageConfig: paneInfo[4],
envelopingConfig: paneInfo[5],
+ testRangeSelectionConfig: paneInfo[6],
});
});
},
pane.get('showFullYAxis'),
pane.get('movingAverageConfig'),
pane.get('envelopingConfig'),
+ pane.get('testRangeSelectionConfig'),
];
}));
},
{
Ember.run.debounce(this, '_updateQueryString', 1000);
}.observes('sharedZoom', 'panes.@each.platform', 'panes.@each.metric', 'panes.@each.selectedItem', 'panes.@each.timeRange',
- 'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig'),
+ 'panes.@each.showFullYAxis', 'panes.@each.movingAverageConfig', 'panes.@each.envelopingConfig', 'panes.@each.testRangeSelectionConfig'),
_updateQueryString: function ()
{
},
{
id: 4,
+ isSegmentation: true,
label: 'Segmentation: Recursive t-test',
description: "Recursively split values into two segments if Welch's t-test detects a statistically significant difference.",
parameterList: [{label: "Minimum segment length", value: 20, min: 5}],
},
{
id: 5,
+ isSegmentation: true,
label: 'Segmentation: Schwarz criterion',
description: 'Adaptive algorithm that maximizes the Schwarz criterion (BIC).',
// Based on Detection of Multiple Change–Points in Multivariate Time Series by Marc Lavielle (July 2006).
},
];
+ this.debuggingTestingRangeNomination = false;
+
+ this.TestRangeSelectionStrategies = [
+ {
+ id: 301,
+ label: "t-test 99% significance",
+ execute: function (values, segmentedValues) {
+ if (!values.length)
+ return [];
+
+ var previousMean = segmentedValues[0];
+ var selectedRanges = new Array;
+ for (var i = 1; i < segmentedValues.length; i++) {
+ var currentMean = segmentedValues[i];
+ if (currentMean == previousMean)
+ continue;
+ var found = false;
+ for (var leftEdge = i - 2, rightEdge = i + 2; leftEdge >= 0 && rightEdge <= values.length; leftEdge--, rightEdge++) {
+ if (segmentedValues[leftEdge] != previousMean || segmentedValues[rightEdge - 1] != currentMean)
+ break;
+ var result = Statistics.computeWelchsT(values, leftEdge, i - leftEdge, values, i, rightEdge - i);
+ if (result.significantlyDifferent) {
+ selectedRanges.push([leftEdge, rightEdge - 1]);
+ found = true;
+ break;
+ }
+ }
+ if (!found && Statistics.debuggingTestingRangeNomination)
+ console.log('Failed to find a testing range at', i, 'changing from', previousValue, 'to', currentValue);
+ previousMean = currentMean;
+ }
+ return selectedRanges;
+ }
+ }
+ ];
+
function createWesternElectricRule(windowSize, minOutlinerCount, limitFactor) {
return function (values, movingAverages, deviation) {
var results = new Array(values.length);
}
return Math.max(valuesAboveLimit, valuesBelowLimit);
}
- window.countValuesOnSameSide = countValuesOnSameSide;
this.AnomalyDetectionStrategy = [
// Western Electric rules: http://en.wikipedia.org/wiki/Western_Electric_rules
git://git.webkit.org / WebKit-https.git / commitdiff