Teach TestFailures how to load, parse, and interpret NRWT test results
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / Builder.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function Builder(name, buildbot) {
27     this.name = name;
28     this.buildbot = buildbot;
29     this._cache = {};
30 }
31
32 Builder.prototype = {
33     buildURL: function(buildName) {
34         return this.buildbot.buildURL(this.name, buildName);
35     },
36
37     failureDiagnosisTextAndURL: function(buildName, testName, testResult) {
38         var urlStem = this.resultsDirectoryURL(buildName) + testName.replace(/\.[^.]+$/, '');
39         var diagnosticInfo = {
40             fail: {
41                 text: 'pretty diff',
42                 url: urlStem + '-pretty-diff.html',
43             },
44             flaky: {
45                 text: 'pretty diff (flaky)',
46                 url: urlStem + '-pretty-diff.html',
47             },
48             timeout: {
49                 text: 'timed out',
50             },
51             crash: {
52                 text: 'crash log',
53                 url: urlStem + '-crash-log.txt',
54             },
55             'webprocess crash': {
56                 text: 'web process crash log',
57                 url: urlStem + '-crash-log.txt',
58             },
59         };
60
61         return diagnosticInfo[testResult.failureType];
62     },
63
64     getBuildNames: function(callback) {
65         this._getBuildNamesFromResultsDirectory(this.buildbot.baseURL + 'results/' + this.name, callback);
66     },
67
68     getMostRecentCompletedBuildNumber: function(callback) {
69         var cacheKey = 'getMostRecentCompletedBuildNumber';
70         if (cacheKey in this._cache) {
71             callback(this._cache[cacheKey]);
72             return;
73         }
74
75         var self = this;
76         getResource(self.buildbot.baseURL + 'json/builders/' + self.name, function(xhr) {
77             var data = JSON.parse(xhr.responseText);
78
79             var currentBuilds = {};
80             if ('currentBuilds' in data)
81                 data.currentBuilds.forEach(function(buildNumber) { currentBuilds[buildNumber] = true });
82
83             for (var i = data.cachedBuilds.length - 1; i >= 0; --i) {
84                 if (data.cachedBuilds[i] in currentBuilds)
85                     continue;
86
87                 self._cache[cacheKey] = data.cachedBuilds[i];
88                 callback(data.cachedBuilds[i]);
89                 return;
90             }
91
92             self._cache[cacheKey] = -1;
93             callback(self._cache[cacheKey]);
94         },
95         function(xhr) {
96             self._cache[cacheKey] = -1;
97             callback(self._cache[cacheKey]);
98         });
99     },
100
101     getNumberOfFailingTests: function(buildNumber, callback) {
102         var cacheKey = this.name + '_getNumberOfFailingTests_' + buildNumber;
103         if (PersistentCache.contains(cacheKey)) {
104             var cachedData = PersistentCache.get(cacheKey);
105             // Old versions of this function used to cache a number instead of an object, so we have
106             // to check to see what type we have.
107             if (typeof cachedData === 'object') {
108                 callback(cachedData.failureCount, cachedData.tooManyFailures);
109                 return;
110             }
111         }
112
113         var result = { failureCount: -1, tooManyFailures: false };
114
115         var self = this;
116         self._getBuildJSON(buildNumber, function(data) {
117             var layoutTestStep = data.steps.findFirst(function(step) { return step.name === 'layout-test'; });
118             if (!layoutTestStep) {
119                 PersistentCache.set(cacheKey, result);
120                 callback(result.failureCount, result.tooManyFailures);
121                 return;
122             }
123
124             if (!('isStarted' in layoutTestStep)) {
125                 // run-webkit-tests never even ran.
126                 PersistentCache.set(cacheKey, result);
127                 callback(result.failureCount, result.tooManyFailures);
128                 return;
129             }
130
131             if (!('results' in layoutTestStep) || layoutTestStep.results[0] === 0) {
132                 // All tests passed.
133                 result.failureCount = 0;
134                 PersistentCache.set(cacheKey, result);
135                 callback(result.failureCount, result.tooManyFailures);
136                 return;
137             }
138
139             if (/^Exiting early/.test(layoutTestStep.results[1][0]))
140                 result.tooManyFailures = true;
141
142             result.failureCount = layoutTestStep.results[1].reduce(function(sum, outputLine) {
143                 var match = /^(\d+)/.exec(outputLine);
144                 if (!match)
145                     return sum;
146                 // Don't count new tests or passes as failures.
147                 if (outputLine.contains('were new') || outputLine.contains('new passes'))
148                     return sum;
149                 return sum + parseInt(match[1], 10);
150             }, 0);
151
152             PersistentCache.set(cacheKey, result);
153             callback(result.failureCount, result.tooManyFailures);
154         });
155     },
156
157     getOldBuildNames: function(callback) {
158         this._getBuildNamesFromResultsDirectory(this.buildbot.baseURL + 'old-results/' + this.name, callback);
159     },
160
161     resultsDirectoryURL: function(buildName) {
162         return this.buildbot.resultsDirectoryURL(this.name, buildName);
163     },
164
165     resultsPageURL: function(buildName) {
166         return this.resultsDirectoryURL(buildName) + 'results.html';
167     },
168
169     _getBuildJSON: function(buildNumber, callback) {
170         var cacheKey = 'getBuildJSON_' + buildNumber;
171         if (cacheKey in this._cache) {
172             callback(this._cache[cacheKey]);
173             return;
174         }
175
176         var self = this;
177         getResource(self.buildbot.baseURL + 'json/builders/' + self.name + '/builds/' + buildNumber, function(xhr) {
178             var data = JSON.parse(xhr.responseText);
179             self._cache[cacheKey] = data;
180             callback(data);
181         });
182     },
183
184     _getBuildNamesFromResultsDirectory: function(directoryURL, callback) {
185         var cacheKey = '_getBuildNamesFromResultsDirectory.' + directoryURL;
186         if (cacheKey in this._cache) {
187             callback(this._cache[cacheKey]);
188             return;
189         }
190
191         var self = this;
192
193         function buildNamesFromDirectoryXHR(xhr) {
194             var root = document.createElement('html');
195             root.innerHTML = xhr.responseText;
196
197             var buildNames = Array.prototype.map.call(root.querySelectorAll('td:first-child > a > b'), function(elem) {
198                 return elem.innerText.replace(/\/$/, '');
199             }).filter(function(filename) {
200                 return self.buildbot.parseBuildName(filename);
201             });
202             buildNames.reverse();
203
204             return buildNames;
205         }
206
207         getResource(directoryURL, function(xhr) {
208             var buildNames = buildNamesFromDirectoryXHR(xhr);
209             self._cache[cacheKey] = buildNames;
210             callback(buildNames);
211         });
212     },
213 };