Teach Buildbot dashboard to parse alternative revision format
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / dashboard / Scripts / BuildbotIteration.js
1 /*
2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 BuildbotIteration = function(queue, id, finished)
27 {
28     BaseObject.call(this);
29
30     console.assert(queue);
31
32     this.queue = queue;
33     this.id = id;
34
35     this.loaded = false;
36
37     this.openSourceRevision = null;
38     this.internalRevision = null;
39
40     this.layoutTestResults = null;
41     this.javascriptTestResults = null;
42     this.apiTestResults = null;
43     this.platformAPITestResults = null;
44     this.pythonTestResults = null;
45     this.perlTestResults = null;
46     this.bindingTestResults = null;
47
48     this._finished = finished;
49 };
50
51 BaseObject.addConstructorFunctions(BuildbotIteration);
52
53 // JSON result values for both individual steps and the whole iteration.
54 BuildbotIteration.SUCCESS = 0;
55 BuildbotIteration.WARNINGS = 1;
56 BuildbotIteration.FAILURE = 2;
57 BuildbotIteration.SKIPPED = 3;
58 BuildbotIteration.EXCEPTION = 4;
59 BuildbotIteration.RETRY = 5;
60
61 BuildbotIteration.Event = {
62     Updated: "updated"
63 };
64
65 // See <http://docs.buildbot.net/0.8.8/manual/cfg-properties.html>.
66 function isMultiCodebaseGotRevisionProperty(property)
67 {
68     return property[0] === "got_revision" && typeof property[1] === "object";
69 }
70
71 function parseRevisionProperty(property, key)
72 {
73     if (!property)
74         return null;
75     var value = property[1];
76     return parseInt(isMultiCodebaseGotRevisionProperty(property) ? value[key] : value, 10);
77 }
78
79 BuildbotIteration.prototype = {
80     constructor: BuildbotIteration,
81     __proto__: BaseObject.prototype,
82
83     get finished()
84     {
85         return this._finished;
86     },
87
88     set finished(x)
89     {
90         this._finished = x;
91     },
92
93     get successful()
94     {
95         return this._result === BuildbotIteration.SUCCESS;
96     },
97
98     get productive()
99     {
100         return this.loaded && this._finished && this._result !== BuildbotIteration.EXCEPTION && this._result !== BuildbotIteration.RETRY;
101     },
102
103     // It is not a real failure if Buildbot itself failed with codes like EXCEPTION or RETRY.
104     get failed()
105     {
106         return this._result === BuildbotIteration.FAILURE;
107     },
108
109     get firstFailedStepName()
110     {
111         if (!this._firstFailedStep)
112             return undefined;
113         return this._firstFailedStep.name;
114     },
115
116     failureLogURL: function(kind)
117     {
118         if (!this.failed)
119             return undefined;
120
121         console.assert(this._firstFailedStep);
122
123         for (var i = 0; i < this._firstFailedStep.logs.length; ++i) {
124             if (this._firstFailedStep.logs[i][0] == kind)
125                 return this._firstFailedStep.logs[i][1];
126         }
127
128         return undefined;
129     },
130
131     get failureLogs()
132     {
133         if (!this.failed)
134             return undefined;
135
136         console.assert(this._firstFailedStep);
137         return this._firstFailedStep.logs;
138     },
139
140     get previousProductiveIteration()
141     {
142         for (var i = 0; i < this.queue.iterations.length - 1; ++i) {
143             if (this.queue.iterations[i] === this) {
144                 while (++i < this.queue.iterations.length) {
145                     var iteration = this.queue.iterations[i];
146                     if (iteration.productive)
147                         return iteration;
148                 }
149                 break;
150             }
151         }
152         return null;
153     },
154
155     update: function()
156     {
157         if (this.loaded && this._finished)
158             return;
159
160         function collectTestResults(data, stepName)
161         {
162             var testStep = data.steps.findFirst(function(step) { return step.name === stepName; });
163             if (!testStep)
164                 return null;
165
166             var testResults = {};
167
168             if (!testStep.isFinished) {
169                 // The step never even ran, or hasn't finish running.
170                 testResults.finished = false;
171                 return testResults;
172             }
173
174             testResults.finished = true;
175
176             if (!testStep.results || !testStep.results[0]) {
177                 // All tests passed.
178                 testResults.allPassed = true;
179                 return testResults;
180             }
181
182             if (/Exiting early/.test(testStep.results[1][0]))
183                 testResults.tooManyFailures = true;
184
185             function resultSummarizer(matchString, sum, outputLine)
186             {
187                 var match = /^(\d+)\s/.exec(outputLine);
188                 if (!match)
189                     return sum;
190                 if (!outputLine.contains(matchString))
191                     return sum;
192                 if (!sum || sum === -1)
193                     sum = 0;
194                 return sum + parseInt(match[1], 10);
195             }
196
197             testResults.failureCount = testStep.results[1].reduce(resultSummarizer.bind(null, "fail"), undefined);
198             testResults.flakeyCount = testStep.results[1].reduce(resultSummarizer.bind(null, "flake"), undefined);
199             testResults.totalLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "total leak"), undefined);
200             testResults.uniqueLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "unique leak"), undefined);
201             testResults.newPassesCount = testStep.results[1].reduce(resultSummarizer.bind(null, "new pass"), undefined);
202             testResults.missingCount = testStep.results[1].reduce(resultSummarizer.bind(null, "missing"), undefined);
203
204             if (!testResults.failureCount && !testResults.flakyCount && !testResults.totalLeakCount && !testResults.uniqueLeakCount && !testResults.newPassesCount && !testResults.missingCount) {
205                 // This step exited with a non-zero exit status, but we didn't find any output about the number of failed tests.
206                 // Something must have gone wrong (e.g., timed out and was killed by buildbot).
207                 testResults.errorOccurred = true;
208             }
209
210             return testResults;
211         }
212
213         JSON.load(this.queue.baseURL + "/builds/" + this.id, function(data) {
214             if (!data || !data.properties)
215                 return;
216
217             // The property got_revision may have the following forms:
218             //
219             // ["got_revision",{"Internal":"1357","WebKitOpenSource":"2468"},"Source"]
220             // OR
221             // ["got_revision","2468_1357","Source"]
222             // OR
223             // ["got_revision","2468","Source"]
224             //
225             // When extracting the OpenSource revision from property got_revision we don't need to check whether the
226             // value of got_revision is a dictionary (represents multiple codebases) or a string literal because we
227             // assume that got_revision contains the OpenSource revision. However, it may not have the Internal
228             // revision. Therefore, we only look at got_revision to extract the Internal revision when it's
229             // a dictionary.
230
231             var openSourceRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "got_revision" || property[0] === "revision" || property[0] === "opensource_got_revision"; });
232             this.openSourceRevision = parseRevisionProperty(openSourceRevisionProperty, "WebKitOpenSource");
233
234             var internalRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "internal_got_revision" || isMultiCodebaseGotRevisionProperty(property); });
235             this.internalRevision = parseRevisionProperty(internalRevisionProperty, "Internal");
236
237             var layoutTestResults = collectTestResults.call(this, data, "layout-test");
238             this.layoutTestResults = layoutTestResults ? new BuildbotTestResults(this, layoutTestResults) : null;
239
240             var javascriptTestResults = collectTestResults.call(this, data, "jscore-test");
241             this.javascriptTestResults = javascriptTestResults ? new BuildbotTestResults(this, javascriptTestResults) : null;
242
243             var apiTestResults = collectTestResults.call(this, data, "run-api-tests");
244             this.apiTestResults = apiTestResults ? new BuildbotTestResults(this, apiTestResults) : null;
245
246             var platformAPITestResults = collectTestResults.call(this, data, "API tests");
247             this.platformAPITestResults = platformAPITestResults ? new BuildbotTestResults(this, platformAPITestResults) : null;
248
249             var pythonTestResults = collectTestResults.call(this, data, "webkitpy-test");
250             this.pythonTestResults = pythonTestResults ? new BuildbotTestResults(this, pythonTestResults) : null;
251
252             var perlTestResults = collectTestResults.call(this, data, "webkitperl-test");
253             this.perlTestResults = perlTestResults ? new BuildbotTestResults(this, perlTestResults) : null;
254
255             var bindingTestResults = collectTestResults.call(this, data, "bindings-generation-tests");
256             this.bindingTestResults = bindingTestResults ? new BuildbotTestResults(this, bindingTestResults) : null;
257
258             this.loaded = true;
259
260             this._firstFailedStep = data.steps.findFirst(function(step) { return step.results[0] === BuildbotIteration.FAILURE; });
261
262             console.assert(data.results === null || typeof data.results === "number");
263             this._result = data.results;
264
265             this.text = data.text.join(" ");
266
267             if (!data.currentStep)
268                 this.finished = true;
269
270             // Update the sorting since it is based on the revisions we just loaded.
271             this.queue.sortIterations();
272
273             this.dispatchEventToListeners(BuildbotIteration.Event.Updated);
274         }.bind(this));
275     },
276
277     loadLayoutTestResults: function(callback)
278     {
279         function collectResults(subtree, predicate)
280         {
281             // Results object is a trie:
282             // directory
283             //   subdirectory
284             //     test1.html
285             //       expected:"PASS"
286             //       actual: "IMAGE"
287             //       report: "REGRESSION"
288             //     test2.html
289             //       expected:"FAIL"
290             //       actual:"TEXT"
291
292             var result = [];
293             for (var key in subtree) {
294                 var value = subtree[key];
295                 console.assert(typeof value === "object");
296                 var isIndividualTest = value.hasOwnProperty("actual") && value.hasOwnProperty("expected");
297                 if (isIndividualTest) {
298                     // Possible values for actual and expected keys: PASS, FAIL, AUDIO, IMAGE, TEXT, IMAGE+TEXT, TIMEOUT, CRASH, MISSING.
299                     // Both actual and expected can be space separated lists. Actual contains two values when retrying a failed test
300                     // gives a different result (retrying may be disabled in tester configuration).
301                     // Possible values for report key (when present): REGRESSION, MISSING, FLAKY.
302
303                     if (predicate(value)) {
304                         var item = {path: key};
305
306                         // FIXME (bug 127186): Crash log URL will be incorrect if crash only happened on retry (e.g. "TEXT CRASH").
307                         // It should point to retries subdirectory, but the information about which attempt failed gets lost here.
308                         if (value.actual.contains("CRASH"))
309                             item.crash = true;
310                         if (value.actual.contains("TIMEOUT"))
311                             item.timeout = true;
312
313                         // FIXME (bug 127186): Similarly, we don't have a good way to present results for something like "TIMEOUT TEXT",
314                         // not even UI wise. For now, only show a diff link if the first attempt has the diff.
315                         if (value.actual.split(" ")[0].contains("TEXT"))
316                             item.has_diff = true;
317
318                         // FIXME (bug 127186): It is particularly unfortunate for image diffs, because we currently only check image results
319                         // on retry (except for reftests), so many times, you will see images on buidbot page, but not on the dashboard.
320                         // FIXME: Find a way to display expected mismatch reftest failures. 
321                         if (value.actual.split(" ")[0].contains("IMAGE") && value.reftest_type != "!=")
322                             item.has_image_diff = true;
323
324                         if (value.has_stderr)
325                             item.has_stderr = true;
326
327                         result.push(item);
328                     }
329
330                 } else {
331                     var nestedTests = collectResults(value, predicate);
332                     for (var i = 0, end = nestedTests.length; i < end; ++i)
333                         nestedTests[i].path = key + "/" + nestedTests[i].path;
334                     result = result.concat(nestedTests);
335                 }
336             }
337
338             return result;
339         }
340
341         JSON.load(this.queue.buildbot.layoutTestFullResultsURLForIteration(this), function(data) {
342             if (data.error) {
343                 console.log(data.error);
344                 callback();
345                 return;
346             }
347
348             this.hasPrettyPatch = data.has_pretty_patch;
349
350             this.layoutTestResults.regressions = collectResults(data.tests, function(info) { return info["report"] === "REGRESSION" });
351             console.assert(data.num_regressions === this.layoutTestResults.regressions.length);
352
353             this.layoutTestResults.flakyTests = collectResults(data.tests, function(info) { return info["report"] === "FLAKY" });
354             console.assert(data.num_flaky === this.layoutTestResults.flakyTests.length);
355
356             this.layoutTestResults.testsWithMissingResults = collectResults(data.tests, function(info) { return info["report"] === "MISSING" });
357             console.assert(data.num_missing === this.layoutTestResults.testsWithMissingResults.length);
358
359             callback();
360         }.bind(this), "ADD_RESULTS");
361     }
362 };