Show t-test results based on individual measurements to analysis task page.
[WebKit-https.git] / Websites / perf.webkit.org / public / v3 / models / test-group.js
index fa41dda..4ac43b4 100644 (file)
@@ -134,25 +134,31 @@ class TestGroup extends LabeledObject {
         return this._buildRequests.some(function (request) { return request.isPending(); });
     }
 
-    compareTestResults(metric, beforeValues, afterValues)
+    compareTestResults(metric, beforeMeasurements, afterMeasurements)
     {
         console.assert(metric);
+        const beforeValues = beforeMeasurements.map((measurment) => measurment.value);
+        const afterValues = afterMeasurements.map((measurement) => measurement.value);
         const beforeMean = Statistics.sum(beforeValues) / beforeValues.length;
         const afterMean = Statistics.sum(afterValues) / afterValues.length;
 
-        var result = {changeType: null, status: 'failed', label: 'Failed', fullLabel: 'Failed', isStatisticallySignificant: false};
+        const result = {changeType: null, status: 'failed', label: 'Failed', fullLabelForMean: 'Failed',
+            isStatisticallySignificantForMean: false, fullLabelForIndividual: 'Failed', isStatisticallySignificantForIndividual: false,
+            probabilityRangeForMean: [null, null], probabilityRangeForIndividual: [null, null]};
 
-        var hasCompleted = this.hasFinished();
+        const hasCompleted = this.hasFinished();
         if (!hasCompleted) {
             if (this.hasStarted()) {
                 result.status = 'running';
                 result.label = 'Running';
-                result.fullLabel = 'Running';
+                result.fullLabelForMean = 'Running';
+                result.fullLabelForIndividual = 'Running';
             } else {
                 console.assert(result.changeType === null);
                 result.status = 'pending';
                 result.label = 'Pending';
-                result.fullLabel = 'Pending';
+                result.fullLabelForMean = 'Pending';
+                result.fullLabelForIndividual = 'Pending';
             }
         }
 
@@ -160,13 +166,25 @@ class TestGroup extends LabeledObject {
             const summary = metric.labelForDifference(beforeMean, afterMean, 'better', 'worse');
             result.changeType = summary.changeType;
             result.label = summary.changeLabel;
-            var isSignificant = Statistics.testWelchsT(beforeValues, afterValues);
-            var significanceLabel = isSignificant ? 'significant' : 'insignificant';
+
+
+            const constructSignificanceLabel = (probabilityRange) => !!probabilityRange.range[0] ? `significant with ${(probabilityRange.range[0] * 100).toFixed()}% probability` : 'insignificant';
+
+            const probabilityRangeForMean = Statistics.probabilityRangeForWelchsT(beforeValues, afterValues);
+            const significanceLabelForMean = constructSignificanceLabel(probabilityRangeForMean);
+            result.fullLabelForMean = `${result.label} (${significanceLabelForMean})`;
+            result.isStatisticallySignificantForMean = !!probabilityRangeForMean.range[0];
+            result.probabilityRangeForMean = probabilityRangeForMean.range;
+
+            const adaptMeasurementToSamples = (measurement) => ({sum: measurement.sum, squareSum: measurement.squareSum, sampleSize: measurement.iterationCount});
+            const probabilityRangeForIndividual = Statistics.probabilityRangeForWelchsTForMultipleSamples(beforeMeasurements.map(adaptMeasurementToSamples), afterMeasurements.map(adaptMeasurementToSamples));
+            const significanceLabelForIndividual = constructSignificanceLabel(probabilityRangeForIndividual);
+            result.fullLabelForIndividual = `${result.label} (${significanceLabelForIndividual})`;
+            result.isStatisticallySignificantForIndividual = !!probabilityRangeForIndividual.range[0];
+            result.probabilityRangeForIndividual = probabilityRangeForIndividual.range;
 
             if (hasCompleted)
-                result.status = isSignificant ? result.changeType : 'unchanged';
-            result.fullLabel = `${result.label} (statistically ${significanceLabel})`;
-            result.isStatisticallySignificant = isSignificant;
+                result.status = result.isStatisticallySignificantForMean ? result.changeType : 'unchanged';
         }
 
         return result;