Teach TestFailures to detect possibly flaky tests and list them separately
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / FlakyLayoutTestDetector.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function FlakyLayoutTestDetector() {
27     this._tests = {};
28 }
29
30 FlakyLayoutTestDetector.prototype = {
31     incorporateTestResults: function(buildName, failingTests, tooManyFailures) {
32         var newFlakyTests = [];
33
34         if (tooManyFailures) {
35             // Something was going horribly wrong during this test run. We shouldn't assume that any
36             // passes/failures are due to flakiness.
37             return newFlakyTests;
38         }
39
40         // Record failing tests.
41         for (var testName in failingTests) {
42             if (!(testName in this._tests)) {
43                 this._tests[testName] = {
44                     state: this._states.LastSeenFailing,
45                     history: [],
46                 };
47             }
48
49             var testData = this._tests[testName];
50             testData.history.push({ build: buildName, result: failingTests[testName] });
51
52             if (testData.state === this._states.LastSeenPassing) {
53                 testData.state = this._states.PossiblyFlaky;
54                 newFlakyTests.push(testName);
55             }
56         }
57
58         // Record passing tests.
59         for (var testName in this._tests) {
60             if (testName in failingTests)
61                 continue;
62
63             var testData = this._tests[testName];
64             testData.history.push({ build: buildName, result: 'pass' });
65
66             if (testData.state === this._states.LastSeenFailing)
67                 testData.state = this._states.LastSeenPassing;
68         }
69
70         return newFlakyTests;
71     },
72
73     flakinessExamples: function(testName) {
74         if (!(testName in this._tests) || this._tests[testName].state !== this._states.PossiblyFlaky)
75             return null;
76
77         var history = this._tests[testName].history;
78
79         var examples = [];
80         for (var i = 0; i < history.length - 1; ++i) {
81             var thisIsPassing = history[i].result === 'pass';
82             var nextIsPassing = history[i + 1].result === 'pass';
83             if (thisIsPassing === nextIsPassing)
84                 continue;
85             var last = examples.last();
86             if (!last || last.build !== history[i].build)
87                 examples.push(history[i]);
88             examples.push(history[i + 1]);
89         }
90
91         return examples;
92     },
93
94     get possiblyFlakyTests() {
95         var self = this;
96         return Object.keys(self._tests).filter(function(testName) { return self._tests[testName].state === self._states.PossiblyFlaky });
97     },
98
99     _states: {
100         LastSeenFailing: 0,
101         LastSeenPassing: 1,
102         PossiblyFlaky: 2,
103     },
104 };