Add a new page to build.webkit.org to help find when tests started failing
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / Builder.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function Builder(name, buildbot) {
27     this.name = name;
28     this.buildbot = buildbot;
29     this._cache = {};
30 }
31
32 Builder.prototype = {
33     buildURL: function(buildName) {
34         return this.buildbot.buildURL(this.name, buildName);
35     },
36
37     failureDiagnosisTextAndURL: function(buildName, testName, failureType) {
38         var urlStem = this.resultsDirectoryURL(buildName) + testName.replace(/\.[^.]+$/, '');
39         var diagnosticInfo = {
40             fail: {
41                 text: 'pretty diff',
42                 url: urlStem + '-pretty-diff.html',
43             },
44             timeout: {
45                 text: 'timed out',
46             },
47             crash: {
48                 text: 'crash log',
49                 url: urlStem + '-crash-log.txt',
50             },
51             'webprocess crash': {
52                 text: 'web process crash log',
53                 url: urlStem + '-crash-log.txt',
54             },
55         };
56
57         return diagnosticInfo[failureType];
58     },
59
60     /*
61      * Preiodically calls callback until all current failures have been explained. Callback is
62      * passed an object like the following:
63      * {
64      *     'r2_1 (1)': {
65      *         'css1/basic/class_as_selector2.html': 'fail',
66      *     },
67      *     'r1_1 (0)': {
68      *         'css1/basic/class_as_selector.html': 'crash',
69      *     },
70      * },
71      * Each build contains just the failures that a) are still occuring on the bots, and b) were new
72      * in that build.
73      */
74     startFetchingBuildHistory: function(callback) {
75         var cacheKey = '_startFetchingBuildHistory';
76         if (!(cacheKey in this._cache))
77             this._cache[cacheKey] = {};
78
79         var history = this._cache[cacheKey];
80
81         var self = this;
82         self._getBuildNames(function(buildNames) {
83             function inner(buildIndex) {
84                 self._incorporateBuildHistory(buildNames, buildIndex, history, function(callAgain) {
85                     callback(history);
86                     if (!callAgain)
87                         return;
88                     var nextIndex = buildIndex + 1;
89                     if (nextIndex >= buildNames.length)
90                         return;
91                     setTimeout(function() { inner(nextIndex) }, 0);
92                 });
93             }
94             inner(0);
95         });
96     },
97
98     resultsDirectoryURL: function(buildName) {
99         return this.buildbot.resultsDirectoryURL(this.name, buildName);
100     },
101
102     _getBuildNames: function(callback) {
103         var cacheKey = '_getBuildNames';
104         if (cacheKey in this._cache) {
105             callback(this._cache[cacheKey]);
106             return;
107         }
108
109         var self = this;
110         getResource(this.buildbot.baseURL + 'results/' + this.name, function(xhr) {
111             var root = document.createElement('html');
112             root.innerHTML = xhr.responseText;
113
114             var buildNames = Array.prototype.map.call(root.querySelectorAll('td:first-child > a > b'), function(elem) {
115                 return elem.innerText.replace(/\/$/, '');
116             }).filter(function(filename) {
117                 return !/\.zip$/.test(filename);
118             });
119             buildNames.reverse();
120
121             self._cache[cacheKey] = buildNames;
122             callback(buildNames);
123         });
124     },
125
126     _getFailingTests: function(buildName, callback, errorCallback) {
127         var cacheKey = '_getFailingTests_' + buildName;
128         if (cacheKey in this._cache) {
129             callback(this._cache[cacheKey]);
130             return;
131         }
132
133         var tests = {};
134         this._cache[cacheKey] = tests;
135
136         var self = this;
137         getResource(self.buildbot.baseURL + 'json/builders/' + self.name + '/builds/' + self.buildbot.parseBuildName(buildName).buildNumber, function(xhr) {
138             var data = JSON.parse(xhr.responseText);
139             var layoutTestStep = data.steps.findFirst(function(step) { return step.name === 'layout-test'; });
140             if (!('isStarted' in layoutTestStep)) {
141                 // run-webkit-tests never even ran.
142                 errorCallback(tests);
143                 return;
144             }
145
146             if (!('results' in layoutTestStep) || layoutTestStep.results[0] === 0) {
147                 // All tests passed.
148                 callback(tests);
149                 return;
150             }
151
152             if (/^Exiting early/.test(layoutTestStep.results[1][0])) {
153                 // Too many tests crashed or timed out. We can't use this test run.
154                 errorCallback(tests);
155                 return;
156             }
157
158             // Find out which tests failed.
159             getResource(self.resultsDirectoryURL(buildName) + 'results.html', function(xhr) {
160                 var root = document.createElement('html');
161                 root.innerHTML = xhr.responseText;
162
163                 function testsForResultTable(regex) {
164                     var paragraph = Array.prototype.findFirst.call(root.querySelectorAll('p'), function(paragraph) {
165                         return regex.test(paragraph.innerText);
166                     });
167                     if (!paragraph)
168                         return [];
169                     var table = paragraph.nextElementSibling;
170                     console.assert(table.nodeName === 'TABLE');
171                     return Array.prototype.map.call(table.querySelectorAll('td:first-child > a'), function(elem) {
172                         return elem.innerText;
173                     });
174                 }
175
176                 testsForResultTable(/did not match expected results/).forEach(function(name) {
177                     tests[name] = 'fail';
178                 });
179                 testsForResultTable(/timed out/).forEach(function(name) {
180                     tests[name] = 'timeout';
181                 });
182                 testsForResultTable(/tool to crash/).forEach(function(name) {
183                     tests[name] = 'crash';
184                 });
185                 testsForResultTable(/Web process to crash/).forEach(function(name) {
186                     tests[name] = 'webprocess crash';
187                 });
188
189                 callback(tests);
190             },
191             function(xhr) {
192                 // We failed to fetch results.html. run-webkit-tests must have aborted early.
193                 errorCallback(tests);
194             });
195         });
196     },
197
198     _incorporateBuildHistory: function(buildNames, buildIndex, history, callback) {
199         var previousBuildName = Object.keys(history).last();
200         var nextBuildName = buildNames[buildIndex];
201
202         this._getFailingTests(nextBuildName, function(tests) {
203             history[nextBuildName] = {};
204
205             for (var testName in tests) {
206                 if (previousBuildName) {
207                     if (!(testName in history[previousBuildName]))
208                         continue;
209                     delete history[previousBuildName][testName];
210                 }
211                 history[nextBuildName][testName] = tests[testName];
212             }
213
214             callback(Object.keys(history[nextBuildName]).length);
215         },
216         function(tests) {
217             // Some tests failed, but we couldn't fetch results.html (perhaps because the test
218             // run aborted early for some reason). Just skip this build entirely.
219             callback(true);
220         });
221     },
222 };