Dashboard should show performance bots.
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / dashboard / Scripts / BuildbotIteration.js
1 /*
2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 BuildbotIteration = function(queue, dataOrID, finished)
27 {
28     BaseObject.call(this);
29
30     console.assert(queue);
31
32     this.queue = queue;
33
34     if (typeof dataOrID === "object") {
35         this._parseData(dataOrID);
36         return;
37     }
38
39     console.assert(typeof dataOrID === "number");
40     this.id = dataOrID;
41
42     this.loaded = false;
43     this.isLoading = false;
44     this._productive = false;
45
46     this.openSourceRevision = null;
47     this.internalRevision = null;
48
49     this.layoutTestResults = null;
50     this.javascriptTestResults = null;
51     this.apiTestResults = null;
52     this.platformAPITestResults = null;
53     this.pythonTestResults = null;
54     this.perlTestResults = null;
55     this.bindingTestResults = null;
56
57     this._finished = finished;
58 };
59
60 BaseObject.addConstructorFunctions(BuildbotIteration);
61
62 // JSON result values for both individual steps and the whole iteration.
63 BuildbotIteration.SUCCESS = 0;
64 BuildbotIteration.WARNINGS = 1;
65 BuildbotIteration.FAILURE = 2;
66 BuildbotIteration.SKIPPED = 3;
67 BuildbotIteration.EXCEPTION = 4;
68 BuildbotIteration.RETRY = 5;
69
70 // If none of these steps ran, then we didn't get any real results, and the iteration was not productive.
71 BuildbotIteration.ProductiveSteps = {
72     "compile-webkit": 1,
73     "build archive": 1,
74     "Build" : 1,
75     "layout-test": 1,
76     "jscore-test": 1,
77     "run-api-tests": 1,
78     "API tests": 1,
79     "webkitpy-test": 1,
80     "webkitperl-test": 1,
81     "bindings-generation-tests": 1,
82     "run Membuster OS Memory": 1,
83     "run scrollperf": 1,
84     "run PLT3": 1,
85     "perf-test": 1
86 };
87
88 BuildbotIteration.Event = {
89     Updated: "updated",
90     UnauthorizedAccess: "unauthorized-access"
91 };
92
93 // See <http://docs.buildbot.net/0.8.8/manual/cfg-properties.html>.
94 function isMultiCodebaseGotRevisionProperty(property)
95 {
96     return property[0] === "got_revision" && typeof property[1] === "object";
97 }
98
99 function parseRevisionProperty(property, key)
100 {
101     if (!property)
102         return null;
103     var value = property[1];
104     return parseInt(isMultiCodebaseGotRevisionProperty(property) ? value[key] : value, 10);
105 }
106
107 BuildbotIteration.prototype = {
108     constructor: BuildbotIteration,
109     __proto__: BaseObject.prototype,
110
111     get finished()
112     {
113         return this._finished;
114     },
115
116     set finished(x)
117     {
118         this._finished = x;
119     },
120
121     get successful()
122     {
123         return this._result === BuildbotIteration.SUCCESS;
124     },
125
126     get productive()
127     {
128         return this._productive;
129     },
130
131     // It is not a real failure if Buildbot itself failed with codes like EXCEPTION or RETRY.
132     get failed()
133     {
134         return this._result === BuildbotIteration.FAILURE;
135     },
136
137     get firstFailedStepName()
138     {
139         if (!this._firstFailedStep)
140             return undefined;
141         return this._firstFailedStep.name;
142     },
143
144     failureLogURL: function(kind)
145     {
146         if (!this.failed)
147             return undefined;
148
149         console.assert(this._firstFailedStep);
150
151         for (var i = 0; i < this._firstFailedStep.logs.length; ++i) {
152             if (this._firstFailedStep.logs[i][0] == kind)
153                 return this._firstFailedStep.logs[i][1];
154         }
155
156         return undefined;
157     },
158
159     get failureLogs()
160     {
161         if (!this.failed)
162             return undefined;
163
164         console.assert(this._firstFailedStep);
165         return this._firstFailedStep.logs;
166     },
167
168     get previousProductiveIteration()
169     {
170         for (var i = 0; i < this.queue.iterations.length - 1; ++i) {
171             if (this.queue.iterations[i] === this) {
172                 while (++i < this.queue.iterations.length) {
173                     var iteration = this.queue.iterations[i];
174                     if (iteration.productive)
175                         return iteration;
176                 }
177                 break;
178             }
179         }
180         return null;
181     },
182
183     _parseData: function(data)
184     {
185         function collectTestResults(data, stepName)
186         {
187             var testStep = data.steps.findFirst(function(step) { return step.name === stepName; });
188             if (!testStep)
189                 return null;
190
191             var testResults = {};
192
193             if (!testStep.isFinished) {
194                 // The step never even ran, or hasn't finished running.
195                 testResults.finished = false;
196                 return testResults;
197             }
198
199             testResults.finished = true;
200
201             if (!testStep.results || testStep.results[0] === BuildbotIteration.SUCCESS) {
202                 // All tests passed.
203                 testResults.allPassed = true;
204                 return testResults;
205             }
206
207             if (/Exiting early/.test(testStep.results[1][0]))
208                 testResults.tooManyFailures = true;
209
210             function resultSummarizer(matchString, sum, outputLine)
211             {
212                 var match = /^(\d+)\s/.exec(outputLine);
213                 if (!match)
214                     return sum;
215                 if (!outputLine.contains(matchString))
216                     return sum;
217                 if (!sum || sum === -1)
218                     sum = 0;
219                 return sum + parseInt(match[1], 10);
220             }
221
222             testResults.failureCount = testStep.results[1].reduce(resultSummarizer.bind(null, "fail"), undefined);
223             testResults.flakeyCount = testStep.results[1].reduce(resultSummarizer.bind(null, "flake"), undefined);
224             testResults.totalLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "total leak"), undefined);
225             testResults.uniqueLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "unique leak"), undefined);
226             testResults.newPassesCount = testStep.results[1].reduce(resultSummarizer.bind(null, "new pass"), undefined);
227             testResults.missingCount = testStep.results[1].reduce(resultSummarizer.bind(null, "missing"), undefined);
228
229             if (!testResults.failureCount && !testResults.flakyCount && !testResults.totalLeakCount && !testResults.uniqueLeakCount && !testResults.newPassesCount && !testResults.missingCount) {
230                 // This step exited with a non-zero exit status, but we didn't find any output about the number of failed tests.
231                 // Something must have gone wrong (e.g., timed out and was killed by buildbot).
232                 testResults.errorOccurred = true;
233             }
234
235             return testResults;
236         }
237
238         console.assert(!this.id || this.id === data.number);
239         this.id = data.number;
240
241         // The property got_revision may have the following forms:
242         //
243         // ["got_revision",{"Internal":"1357","WebKitOpenSource":"2468"},"Source"]
244         // OR
245         // ["got_revision","2468_1357","Source"]
246         // OR
247         // ["got_revision","2468","Source"]
248         //
249         // When extracting the OpenSource revision from property got_revision we don't need to check whether the
250         // value of got_revision is a dictionary (represents multiple codebases) or a string literal because we
251         // assume that got_revision contains the OpenSource revision. However, it may not have the Internal
252         // revision. Therefore, we only look at got_revision to extract the Internal revision when it's
253         // a dictionary.
254
255         var openSourceRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "got_revision" || property[0] === "revision" || property[0] === "opensource_got_revision"; });
256         this.openSourceRevision = parseRevisionProperty(openSourceRevisionProperty, "WebKit");
257
258         var internalRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "internal_got_revision" || isMultiCodebaseGotRevisionProperty(property); });
259         this.internalRevision = parseRevisionProperty(internalRevisionProperty, "Internal");
260
261         this.startTime = new Date(data.times[0] * 1000);
262         this.endTime = new Date(data.times[1] * 1000);
263
264         var layoutTestResults = collectTestResults.call(this, data, "layout-test");
265         this.layoutTestResults = layoutTestResults ? new BuildbotTestResults(this, layoutTestResults) : null;
266
267         var javascriptTestResults = collectTestResults.call(this, data, "jscore-test");
268         this.javascriptTestResults = javascriptTestResults ? new BuildbotTestResults(this, javascriptTestResults) : null;
269
270         var apiTestResults = collectTestResults.call(this, data, "run-api-tests");
271         this.apiTestResults = apiTestResults ? new BuildbotTestResults(this, apiTestResults) : null;
272
273         var platformAPITestResults = collectTestResults.call(this, data, "API tests");
274         this.platformAPITestResults = platformAPITestResults ? new BuildbotTestResults(this, platformAPITestResults) : null;
275
276         var pythonTestResults = collectTestResults.call(this, data, "webkitpy-test");
277         this.pythonTestResults = pythonTestResults ? new BuildbotTestResults(this, pythonTestResults) : null;
278
279         var perlTestResults = collectTestResults.call(this, data, "webkitperl-test");
280         this.perlTestResults = perlTestResults ? new BuildbotTestResults(this, perlTestResults) : null;
281
282         var bindingTestResults = collectTestResults.call(this, data, "bindings-generation-tests");
283         this.bindingTestResults = bindingTestResults ? new BuildbotTestResults(this, bindingTestResults) : null;
284
285         this.loaded = true;
286
287         this._firstFailedStep = data.steps.findFirst(function(step) { return step.results[0] === BuildbotIteration.FAILURE; });
288
289         console.assert(data.results === null || typeof data.results === "number");
290         this._result = data.results;
291
292         this.text = data.text.join(" ");
293
294         if (!data.currentStep)
295             this.finished = true;
296
297         this._productive = this._finished && this._result !== BuildbotIteration.EXCEPTION && this._result !== BuildbotIteration.RETRY;
298         if (this._productive) {
299             var finishedAnyProductiveStep = false;
300             for (var i = 0; i < data.steps.length; ++i) {
301                 var step = data.steps[i];
302                 if (!step.isFinished)
303                     break;
304                 if (step.name in BuildbotIteration.ProductiveSteps) {
305                     finishedAnyProductiveStep = true;
306                     break;
307                 }
308             }
309             this._productive = finishedAnyProductiveStep;
310         }
311
312         // Update the sorting since it is based on the revisions we just loaded.
313         this.queue.sortIterations();
314     },
315
316     _updateWithData: function(data)
317     {
318         if (this.loaded && this._finished)
319             return;
320
321         this._parseData(data);
322         this.dispatchEventToListeners(BuildbotIteration.Event.Updated);
323     },
324
325     update: function()
326     {
327         if (this.loaded && this._finished)
328             return;
329
330         if (this.queue.buildbot.needsAuthentication && this.queue.buildbot.authenticationStatus === Buildbot.AuthenticationStatus.InvalidCredentials)
331             return;
332
333         if (this.isLoading)
334             return;
335
336         this.isLoading = true;
337
338         JSON.load(this.queue.baseURL + "/builds/" + this.id, function(data) {
339             this.isLoading = false;
340             this.queue.buildbot.isAuthenticated = true;
341             if (!data || !data.properties)
342                 return;
343
344             this._updateWithData(data);
345         }.bind(this),
346         function(data) {
347             this.isLoading = false;
348             if (data.errorType === JSON.LoadError && data.errorHTTPCode === 401) {
349                 this.queue.buildbot.isAuthenticated = false;
350                 this.dispatchEventToListeners(BuildbotIteration.Event.UnauthorizedAccess);
351             }
352         }.bind(this), {withCredentials: this.queue.buildbot.needsAuthentication});
353     },
354
355     loadLayoutTestResults: function(callback)
356     {
357         if (this.queue.buildbot.needsAuthentication && this.queue.buildbot.authenticationStatus === Buildbot.AuthenticationStatus.InvalidCredentials)
358             return;
359
360         function collectResults(subtree, predicate)
361         {
362             // Results object is a trie:
363             // directory
364             //   subdirectory
365             //     test1.html
366             //       expected:"PASS"
367             //       actual: "IMAGE"
368             //       report: "REGRESSION"
369             //     test2.html
370             //       expected:"FAIL"
371             //       actual:"TEXT"
372
373             var result = [];
374             for (var key in subtree) {
375                 var value = subtree[key];
376                 console.assert(typeof value === "object");
377                 var isIndividualTest = value.hasOwnProperty("actual") && value.hasOwnProperty("expected");
378                 if (isIndividualTest) {
379                     // Possible values for actual and expected keys: PASS, FAIL, AUDIO, IMAGE, TEXT, IMAGE+TEXT, TIMEOUT, CRASH, MISSING.
380                     // Both actual and expected can be space separated lists. Actual contains two values when retrying a failed test
381                     // gives a different result (retrying may be disabled in tester configuration).
382                     // Possible values for report key (when present): REGRESSION, MISSING, FLAKY.
383
384                     if (predicate(value)) {
385                         var item = {path: key};
386
387                         // FIXME (bug 127186): Crash log URL will be incorrect if crash only happened on retry (e.g. "TEXT CRASH").
388                         // It should point to retries subdirectory, but the information about which attempt failed gets lost here.
389                         if (value.actual.contains("CRASH"))
390                             item.crash = true;
391                         if (value.actual.contains("TIMEOUT"))
392                             item.timeout = true;
393
394                         // FIXME (bug 127186): Similarly, we don't have a good way to present results for something like "TIMEOUT TEXT",
395                         // not even UI wise. For now, only show a diff link if the first attempt has the diff.
396                         if (value.actual.split(" ")[0].contains("TEXT"))
397                             item.has_diff = true;
398
399                         // FIXME (bug 127186): It is particularly unfortunate for image diffs, because we currently only check image results
400                         // on retry (except for reftests), so many times, you will see images on buidbot page, but not on the dashboard.
401                         // FIXME: Find a way to display expected mismatch reftest failures. 
402                         if (value.actual.split(" ")[0].contains("IMAGE") && value.reftest_type != "!=")
403                             item.has_image_diff = true;
404
405                         if (value.has_stderr)
406                             item.has_stderr = true;
407
408                         result.push(item);
409                     }
410
411                 } else {
412                     var nestedTests = collectResults(value, predicate);
413                     for (var i = 0, end = nestedTests.length; i < end; ++i)
414                         nestedTests[i].path = key + "/" + nestedTests[i].path;
415                     result = result.concat(nestedTests);
416                 }
417             }
418
419             return result;
420         }
421
422         JSON.load(this.queue.buildbot.layoutTestFullResultsURLForIteration(this), function(data) {
423             this.queue.buildbot.isAuthenticated = true;
424             this.hasPrettyPatch = data.has_pretty_patch;
425
426             this.layoutTestResults.regressions = collectResults(data.tests, function(info) { return info["report"] === "REGRESSION" });
427             console.assert(data.num_regressions === this.layoutTestResults.regressions.length);
428
429             this.layoutTestResults.flakyTests = collectResults(data.tests, function(info) { return info["report"] === "FLAKY" });
430             console.assert(data.num_flaky === this.layoutTestResults.flakyTests.length);
431
432             this.layoutTestResults.testsWithMissingResults = collectResults(data.tests, function(info) { return info["report"] === "MISSING" });
433             console.assert(data.num_missing === this.layoutTestResults.testsWithMissingResults.length);
434
435             callback();
436         }.bind(this),
437         function(data) {
438             if (data.errorType === JSON.LoadError && data.errorHTTPCode === 401) {
439                 this.queue.buildbot.isAuthenticated = false;
440                 this.dispatchEventToListeners(BuildbotIteration.Event.UnauthorizedAccess);
441             }
442             console.log(data.error);
443             callback();
444         }.bind(this), {jsonpCallbackName: "ADD_RESULTS", withCredentials: this.queue.buildbot.needsAuthentication});
445     }
446 };