Teach TestFailures to detect possibly flaky tests and list them separately
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / LayoutTestHistoryAnalyzer.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function LayoutTestHistoryAnalyzer(builder) {
27     this._builder = builder;
28     this._flakinessDetector = new FlakyLayoutTestDetector();
29     this._history = {};
30     this._loader = new LayoutTestResultsLoader(builder);
31     this._testRunsSinceLastInterestingChange = 0;
32 }
33
34 LayoutTestHistoryAnalyzer.prototype = {
35     /*
36      * Periodically calls callback until all current failures have been explained. Callback is
37      * passed an object like the following:
38      * {
39      *     'history': {
40      *         'r12347 (681)': {
41      *             'tooManyFailures': false,
42      *             'tests': {
43      *                 'css1/basic/class_as_selector2.html': 'fail',
44      *             },
45      *         },
46      *         'r12346 (680)': {
47      *             'tooManyFailures': false,
48      *             'tests': {},
49      *         },
50      *         'r12345 (679)': {
51      *             'tooManyFailures': false,
52      *             'tests': {
53      *                 'css1/basic/class_as_selector.html': 'crash',
54      *             },
55      *         },
56      *     },
57      *     'possiblyFlaky': {
58      *         'fast/workers/worker-test.html': [
59      *             { 'build': 'r12345 (679)', 'result': 'pass' },
60      *             { 'build': 'r12344 (678)', 'result': 'fail' },
61      *             { 'build': 'r12340 (676)', 'result': 'fail' },
62      *             { 'build': 'r12338 (675)', 'result': 'pass' },
63      *         ],
64      *     },
65      * }
66      * Each build contains just the failures that a) are still occurring on the bots, and b) were new
67      * in that build.
68      */
69     start: function(callback) {
70         var self = this;
71         self._builder.getBuildNames(function(buildNames) {
72             function inner(buildIndex) {
73                 self._incorporateBuildHistory(buildNames, buildIndex, function(callAgain) {
74                     var nextIndex = buildIndex + 1;
75                     if (nextIndex >= buildNames.length)
76                         callAgain = false;
77                     var data = {
78                         history: self._history,
79                         possiblyFlaky: {},
80                     };
81                     self._flakinessDetector.possiblyFlakyTests.forEach(function(testName) {
82                         data.possiblyFlaky[testName] = self._flakinessDetector.flakinessExamples(testName);
83                     });
84                     callback(data, callAgain);
85                     if (!callAgain)
86                         return;
87                     setTimeout(function() { inner(nextIndex) }, 0);
88                 });
89             }
90             inner(0);
91         });
92     },
93
94     _incorporateBuildHistory: function(buildNames, buildIndex, callback) {
95         var previousBuildName = Object.keys(this._history).last();
96         var nextBuildName = buildNames[buildIndex];
97
98         var self = this;
99         self._loader.start(nextBuildName, function(tests, tooManyFailures) {
100             ++self._testRunsSinceLastInterestingChange;
101
102             self._history[nextBuildName] = {
103                 tooManyFailures: tooManyFailures,
104                 tests: {},
105             };
106
107             var newFlakyTests = self._flakinessDetector.incorporateTestResults(nextBuildName, tests, tooManyFailures);
108             if (newFlakyTests.length) {
109                 self._testRunsSinceLastInterestingChange = 0;
110                 // Remove all possibly flaky tests from the failure history, since when they failed
111                 // is no longer meaningful.
112                 newFlakyTests.forEach(function(testName) {
113                     for (var buildName in self._history)
114                         delete self._history[buildName].tests[testName];
115                 });
116             }
117
118             for (var testName in tests) {
119                 if (previousBuildName) {
120                     if (!(testName in self._history[previousBuildName].tests))
121                         continue;
122                     delete self._history[previousBuildName].tests[testName];
123                 }
124                 self._history[nextBuildName].tests[testName] = tests[testName];
125             }
126
127             var previousUnexplainedFailuresCount = previousBuildName ? Object.keys(self._history[previousBuildName].tests).length : 0;
128             var unexplainedFailuresCount = Object.keys(self._history[nextBuildName].tests).length;
129
130             if (previousUnexplainedFailuresCount && !unexplainedFailuresCount)
131                 self._testRunsSinceLastInterestingChange = 0;
132
133             const minimumRequiredTestRunsWithoutInterestingChanges = 5;
134             callback(unexplainedFailuresCount || self._testRunsSinceLastInterestingChange < minimumRequiredTestRunsWithoutInterestingChanges);
135         },
136         function(tests) {
137             // Some tests failed, but we couldn't fetch results.html (perhaps because the test
138             // run aborted early for some reason). Just skip this build entirely.
139             callback(true);
140         });
141     },
142 };