Teach TestFailures to detect possibly flaky tests and list them separately
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / ViewController.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function ViewController(buildbot, bugzilla, trac) {
27     this._buildbot = buildbot;
28     this._bugzilla = bugzilla;
29     this._trac = trac;
30
31     var self = this;
32     addEventListener('load', function() { self.loaded() }, false);
33     addEventListener('hashchange', function() { self.parseHash(location.hash) }, false);
34 }
35
36 ViewController.prototype = {
37     loaded: function() {
38         this.parseHash(location.hash);
39     },
40
41     parseHash: function(hash) {
42         var match = /#\/(.*)/.exec(hash);
43         if (match)
44             this._displayBuilder(this._buildbot.builderNamed(decodeURIComponent(match[1])));
45         else
46             this._displayTesters();
47     },
48
49     _displayBuilder: function(builder) {
50         var self = this;
51         var lastDisplay = 0;
52         (new LayoutTestHistoryAnalyzer(builder)).start(function(data, stillFetchingData) {
53             var list = document.createElement('ol');
54             list.id = 'failure-history';
55             Object.keys(data.history).forEach(function(buildName, buildIndex, buildNameArray) {
56                 var failingTestNames = Object.keys(data.history[buildName].tests);
57                 if (!failingTestNames.length)
58                     return;
59
60                 var item = document.createElement('li');
61                 list.appendChild(item);
62
63                 var testList = document.createElement('ol');
64                 item.appendChild(testList);
65
66                 testList.className = 'test-list';
67                 for (var testName in data.history[buildName].tests) {
68                     var testItem = document.createElement('li');
69                     testItem.appendChild(self._domForFailedTest(builder, buildName, testName, data.history[buildName].tests[testName]));
70                     testList.appendChild(testItem);
71                 }
72
73                 if (data.history[buildName].tooManyFailures) {
74                     var p = document.createElement('p');
75                     p.className = 'info';
76                     p.appendChild(document.createTextNode('run-webkit-tests exited early due to too many failures/crashes/timeouts'));
77                     item.appendChild(p);
78                 }
79
80                 var passingBuildName;
81                 if (buildIndex + 1 < buildNameArray.length)
82                     passingBuildName = buildNameArray[buildIndex + 1];
83
84                 item.appendChild(self._domForRegressionRange(builder, passingBuildName, buildName));
85
86                 if (passingBuildName || !stillFetchingData)
87                     item.appendChild(self._domForNewAndExistingBugs(builder, buildName, passingBuildName, failingTestNames));
88             });
89
90             var header = document.createElement('h1');
91             header.appendChild(document.createTextNode(builder.name));
92             document.body.innerHTML = '';
93             document.title = builder.name;
94             document.body.appendChild(header);
95             document.body.appendChild(list);
96             document.body.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky));
97
98             if (!stillFetchingData)
99                 PersistentCache.prune();
100         });
101     },
102
103     _displayTesters: function() {
104         var list = document.createElement('ul');
105         var latestBuildInfos = [];
106
107         function updateList() {
108             latestBuildInfos.sort(function(a, b) { return a.tester.name.localeCompare(b.tester.name) });
109             while (list.firstChild)
110                 list.removeChild(list.firstChild);
111             latestBuildInfos.forEach(function(buildInfo) {
112                 var link = document.createElement('a');
113                 link.href = '#/' + buildInfo.tester.name;
114                 link.appendChild(document.createTextNode(buildInfo.tester.name));
115
116                 var item = document.createElement('li');
117                 item.appendChild(link);
118                 if (buildInfo.tooManyFailures)
119                     item.appendChild(document.createTextNode(' (too many failures/crashes/timeouts)'));
120                 else
121                     item.appendChild(document.createTextNode(' (' + buildInfo.failureCount + ' failing test' + (buildInfo.failureCount > 1 ? 's' : '') + ')'));
122                 list.appendChild(item);
123             });
124         }
125
126         this._buildbot.getTesters(function(testers) {
127             testers.forEach(function(tester) {
128                 tester.getMostRecentCompletedBuildNumber(function(buildNumber) {
129                     if (buildNumber < 0)
130                         return;
131                     tester.getNumberOfFailingTests(buildNumber, function(failureCount, tooManyFailures) {
132                         if (failureCount <= 0)
133                             return;
134                         latestBuildInfos.push({ tester: tester, failureCount: failureCount, tooManyFailures: tooManyFailures });
135                         updateList();
136                     });
137                 });
138             });
139
140             document.body.innerHTML = '';
141             document.title = 'Testers';
142             document.body.appendChild(list);
143         });
144     },
145
146     _domForRegressionRange: function(builder, passingBuildName, failingBuildName) {
147         var result = document.createDocumentFragment();
148
149         var dlItems = [
150             [document.createTextNode('Failed'), this._domForBuildName(builder, failingBuildName)],
151         ];
152         if (passingBuildName)
153             dlItems.push([document.createTextNode('Passed'), this._domForBuildName(builder, passingBuildName)]);
154         result.appendChild(createDefinitionList(dlItems));
155
156         if (!passingBuildName)
157             return result;
158
159         var firstSuspectRevision = this._buildbot.parseBuildName(passingBuildName).revision + 1;
160         var lastSuspectRevision = this._buildbot.parseBuildName(failingBuildName).revision;
161
162         if (firstSuspectRevision === lastSuspectRevision)
163             return result;
164
165         var link = document.createElement('a');
166         result.appendChild(link);
167
168         link.href = this._trac.logURL('trunk', firstSuspectRevision, lastSuspectRevision);
169         link.appendChild(document.createTextNode('View regression range in Trac'));
170
171         return result;
172     },
173
174     _domForBuildName: function(builder, buildName) {
175         var parsed = this._buildbot.parseBuildName(buildName);
176
177         var sourceLink = document.createElement('a');
178         sourceLink.href = 'http://trac.webkit.org/changeset/' + parsed.revision;
179         sourceLink.appendChild(document.createTextNode('r' + parsed.revision));
180
181         var buildLink = document.createElement('a');
182         buildLink.href = builder.buildURL(parsed.buildNumber);
183         buildLink.appendChild(document.createTextNode(parsed.buildNumber));
184
185         var resultsLink = document.createElement('a');
186         resultsLink.href = builder.resultsPageURL(buildName);
187         resultsLink.appendChild(document.createTextNode('results.html'));
188
189         var result = document.createDocumentFragment();
190         result.appendChild(sourceLink);
191         result.appendChild(document.createTextNode(' ('));
192         result.appendChild(buildLink);
193         result.appendChild(document.createTextNode(') ('));
194         result.appendChild(resultsLink);
195         result.appendChild(document.createTextNode(')'));
196
197         return result;
198     },
199
200     _domForFailedTest: function(builder, buildName, testName, failureType) {
201         var result = document.createDocumentFragment();
202         result.appendChild(document.createTextNode(testName));
203         result.appendChild(document.createTextNode(' ('));
204         result.appendChild(this._domForFailureDiagnosis(builder, buildName, testName, failureType));
205         result.appendChild(document.createTextNode(')'));
206         return result;
207     },
208
209     _domForFailureDiagnosis: function(builder, buildName, testName, failureType) {
210         var diagnosticInfo = builder.failureDiagnosisTextAndURL(buildName, testName, failureType);
211         if (!diagnosticInfo)
212             return document.createTextNode(failureType);
213
214         var textNode = document.createTextNode(diagnosticInfo.text);
215         if (!('url' in diagnosticInfo))
216             return textNode;
217
218         var link = document.createElement('a');
219         link.href = diagnosticInfo.url;
220         link.appendChild(textNode);
221         return link;
222     },
223
224     _domForNewAndExistingBugs: function(tester, failingBuildName, passingBuildName, failingTests) {
225         var result = document.createDocumentFragment();
226
227         if (!this._bugzilla)
228             return result;
229
230         var container = document.createElement('p');
231         result.appendChild(container);
232
233         container.className = 'existing-and-new-bugs';
234
235         var bugsContainer = document.createElement('div');
236         container.appendChild(bugsContainer);
237
238         bugsContainer.appendChild(document.createTextNode('Searching for bugs related to ' + (failingTests.length > 1 ? 'these tests' : 'this test') + '\u2026'));
239
240         this._bugzilla.quickSearch('ALL ' + failingTests.join('|'), function(bugs) {
241             if (!bugs.length) {
242                 bugsContainer.parentNode.removeChild(bugsContainer);
243                 return;
244             }
245
246             while (bugsContainer.firstChild)
247                 bugsContainer.removeChild(bugsContainer.firstChild);
248
249             bugsContainer.appendChild(document.createTextNode('Existing bugs related to ' + (failingTests.length > 1 ? 'these tests' : 'this test') + ':'));
250
251             var list = document.createElement('ul');
252             bugsContainer.appendChild(list);
253
254             list.className = 'existing-bugs-list';
255
256             function bugToListItem(bug) {
257                 var link = document.createElement('a');
258                 link.href = bug.url;
259                 link.appendChild(document.createTextNode(bug.title));
260
261                 var item = document.createElement('li');
262                 item.appendChild(link);
263
264                 return item;
265             }
266
267             var openBugs = bugs.filter(function(bug) { return Bugzilla.isOpenStatus(bug.status) });
268             var closedBugs = bugs.filter(function(bug) { return !Bugzilla.isOpenStatus(bug.status) });
269
270             list.appendChildren(openBugs.map(bugToListItem));
271
272             if (!closedBugs.length)
273                 return;
274
275             var item = document.createElement('li');
276             list.appendChild(item);
277
278             item.appendChild(document.createTextNode('Closed bugs:'));
279
280             var closedList = document.createElement('ul');
281             item.appendChild(closedList);
282
283             closedList.appendChildren(closedBugs.map(bugToListItem));
284         });
285
286         var parsedFailingBuildName = this._buildbot.parseBuildName(failingBuildName);
287         var regressionRangeString = 'r' + parsedFailingBuildName.revision;
288         if (passingBuildName) {
289             var parsedPassingBuildName = this._buildbot.parseBuildName(passingBuildName);
290             if (parsedFailingBuildName.revision - parsedPassingBuildName.revision > 1)
291                 regressionRangeString = 'r' + parsedPassingBuildName.revision + '-' + regressionRangeString;
292         }
293
294         // FIXME: Some of this code should move into a new method on the Bugzilla class.
295
296         // FIXME: When a newly-added test has been failing since its introduction, it isn't really a
297         // "regression". We should use a different title and keywords in that case.
298         // <http://webkit.org/b/61645>
299
300         var titlePrefix = 'REGRESSION (' + regressionRangeString + '): ';
301         var titleSuffix = ' failing on ' + tester.name;
302         var title = titlePrefix + failingTests.join(', ') + titleSuffix;
303         if (title.length > Bugzilla.maximumBugTitleLength) {
304             var pathPrefix = longestCommonPathPrefix(failingTests);
305             if (pathPrefix)
306                 title = titlePrefix + failingTests.length + ' ' + pathPrefix + ' tests' + titleSuffix;
307             if (title.length > Bugzilla.maximumBugTitleLength)
308                 title = titlePrefix + failingTests.length + ' tests' + titleSuffix;
309         }
310         console.assert(title.length <= Bugzilla.maximumBugTitleLength);
311
312         var firstSuspectRevision = parsedPassingBuildName ? parsedPassingBuildName.revision + 1 : parsedFailingBuildName.revision;
313         var lastSuspectRevision = parsedFailingBuildName.revision;
314
315         var endOfFirstSentence;
316         if (passingBuildName) {
317             endOfFirstSentence = 'started failing on ' + tester.name;
318             if (firstSuspectRevision === lastSuspectRevision)
319                 endOfFirstSentence += ' in r' + firstSuspectRevision + ' <' + this._trac.changesetURL(firstSuspectRevision) + '>';
320             else
321                 endOfFirstSentence += ' between r' + firstSuspectRevision + ' and r' + lastSuspectRevision + ' (inclusive)';
322         } else
323             endOfFirstSentence = (failingTests.length === 1 ? 'has' : 'have') + ' been failing on ' + tester.name + ' since at least r' + firstSuspectRevision + ' <' + this._trac.changesetURL(firstSuspectRevision) + '>';
324
325         var description;
326         if (failingTests.length === 1)
327             description = failingTests[0] + ' ' + endOfFirstSentence + '.\n\n';
328         else if (failingTests.length === 2)
329             description = failingTests.join(' and ') + ' ' + endOfFirstSentence + '.\n\n';
330         else {
331             description = 'The following tests ' + endOfFirstSentence + ':\n\n'
332                 + failingTests.map(function(test) { return '    ' + test }).join('\n')
333                 + '\n\n';
334         }
335         if (firstSuspectRevision !== lastSuspectRevision)
336             description += this._trac.logURL('trunk', firstSuspectRevision, lastSuspectRevision) + '\n\n';
337         if (passingBuildName)
338             description += encodeURI(tester.resultsPageURL(passingBuildName)) + ' passed\n';
339         var failingResultsHTML = tester.resultsPageURL(failingBuildName);
340         description += encodeURI(failingResultsHTML) + ' failed\n';
341
342         var formData = {
343             product: 'WebKit',
344             version: '528+ (Nightly build)',
345             component: 'Tools / Tests',
346             keywords: 'LayoutTestFailure, MakingBotsRed, Regression',
347             short_desc: title,
348             comment: description,
349             bug_file_loc: failingResultsHTML,
350         };
351
352         if (/Windows/.test(tester.name)) {
353             formData.rep_platform = 'PC';
354             if (/Windows 7/.test(tester.name))
355                 formData.op_sys = 'Windows 7';
356             else if (/Windows XP/.test(tester.name))
357                 formData.op_sys = 'Windows XP';
358         } else if (/Leopard/.test(tester.name)) {
359             formData.rep_platform = 'Macintosh';
360             if (/SnowLeopard/.test(tester.name))
361                 formData.op_sys = 'Mac OS X 10.6';
362             else
363                 formData.op_sys = 'Mac OS X 10.5';
364         }
365
366         var form = document.createElement('form');
367         result.appendChild(form);
368         form.className = 'new-bug-form';
369         form.method = 'POST';
370         form.action = this._bugzilla.baseURL + 'enter_bug.cgi';
371         form.target = '_blank';
372
373         for (var key in formData) {
374             var input = document.createElement('input');
375             input.type = 'hidden';
376             input.name = key;
377             input.value = formData[key];
378             form.appendChild(input);
379         }
380
381         var link = document.createElement('a');
382         container.appendChild(link);
383
384         link.addEventListener('click', function(event) { form.submit(); event.preventDefault(); });
385         link.href = '#';
386         link.appendChild(document.createTextNode('File bug for ' + (failingTests.length > 1 ? 'these failures' : 'this failure')));
387
388         return result;
389     },
390
391     _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData) {
392         var result = document.createDocumentFragment();
393         var flakyTests = Object.keys(possiblyFlakyTestData);
394         if (!flakyTests.length)
395             return result;
396
397         var flakyHeader = document.createElement('h2');
398         result.appendChild(flakyHeader);
399         flakyHeader.appendChild(document.createTextNode('Possibly Flaky Tests'));
400
401         var flakyList = document.createElement('ol');
402         result.appendChild(flakyList);
403
404         var self = this;
405         flakyList.appendChildren(sorted(flakyTests).map(function(testName) {
406             var item = document.createElement('li');
407             item.appendChild(document.createTextNode(testName));
408             var historyList = document.createElement('ol');
409             item.appendChild(historyList);
410             historyList.appendChildren(possiblyFlakyTestData[testName].map(function(historyItem) {
411                 var item = document.createElement('li');
412                 item.appendChild(self._domForBuildName(builder, historyItem.build));
413                 item.appendChild(document.createTextNode(': '));
414                 item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
415                 return item;
416             }));
417             return item;
418         }));
419
420         return result;
421     },
422 };