a52b71283d8a29bdb84bcba8eb3cfe7a69f941e6
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / dashboard / Scripts / BuildbotIteration.js
1 /*
2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 BuildbotIteration = function(queue, dataOrID, finished)
27 {
28     BaseObject.call(this);
29
30     console.assert(queue);
31
32     this.queue = queue;
33
34     if (typeof dataOrID === "object") {
35         this._parseData(dataOrID);
36         return;
37     }
38
39     console.assert(typeof dataOrID === "number");
40     this.id = dataOrID;
41
42     this.loaded = false;
43     this.isLoading = false;
44     this._productive = false;
45
46     this.openSourceRevision = null;
47     this.internalRevision = null;
48
49     this.layoutTestResults = null;
50     this.javascriptTestResults = null;
51     this.apiTestResults = null;
52     this.platformAPITestResults = null;
53     this.pythonTestResults = null;
54     this.perlTestResults = null;
55     this.bindingTestResults = null;
56
57     this._finished = finished;
58 };
59
60 BaseObject.addConstructorFunctions(BuildbotIteration);
61
62 // JSON result values for both individual steps and the whole iteration.
63 BuildbotIteration.SUCCESS = 0;
64 BuildbotIteration.WARNINGS = 1;
65 BuildbotIteration.FAILURE = 2;
66 BuildbotIteration.SKIPPED = 3;
67 BuildbotIteration.EXCEPTION = 4;
68 BuildbotIteration.RETRY = 5;
69
70 // If none of these steps ran, then we didn't get any real results, and the iteration was not productive.
71 BuildbotIteration.ProductiveSteps = {
72     "compile-webkit": 1,
73     "build archive": 1,
74     "Build" : 1,
75     "layout-test": 1,
76     "jscore-test": 1,
77     "run-api-tests": 1,
78     "API tests": 1,
79     "webkitpy-test": 1,
80     "webkitperl-test": 1,
81     "bindings-generation-tests": 1,
82 };
83
84 BuildbotIteration.Event = {
85     Updated: "updated",
86     UnauthorizedAccess: "unauthorized-access"
87 };
88
89 // See <http://docs.buildbot.net/0.8.8/manual/cfg-properties.html>.
90 function isMultiCodebaseGotRevisionProperty(property)
91 {
92     return property[0] === "got_revision" && typeof property[1] === "object";
93 }
94
95 function parseRevisionProperty(property, key)
96 {
97     if (!property)
98         return null;
99     var value = property[1];
100     return parseInt(isMultiCodebaseGotRevisionProperty(property) ? value[key] : value, 10);
101 }
102
103 BuildbotIteration.prototype = {
104     constructor: BuildbotIteration,
105     __proto__: BaseObject.prototype,
106
107     get finished()
108     {
109         return this._finished;
110     },
111
112     set finished(x)
113     {
114         this._finished = x;
115     },
116
117     get successful()
118     {
119         return this._result === BuildbotIteration.SUCCESS;
120     },
121
122     get productive()
123     {
124         return this._productive;
125     },
126
127     // It is not a real failure if Buildbot itself failed with codes like EXCEPTION or RETRY.
128     get failed()
129     {
130         return this._result === BuildbotIteration.FAILURE;
131     },
132
133     get firstFailedStepName()
134     {
135         if (!this._firstFailedStep)
136             return undefined;
137         return this._firstFailedStep.name;
138     },
139
140     failureLogURL: function(kind)
141     {
142         if (!this.failed)
143             return undefined;
144
145         console.assert(this._firstFailedStep);
146
147         for (var i = 0; i < this._firstFailedStep.logs.length; ++i) {
148             if (this._firstFailedStep.logs[i][0] == kind)
149                 return this._firstFailedStep.logs[i][1];
150         }
151
152         return undefined;
153     },
154
155     get failureLogs()
156     {
157         if (!this.failed)
158             return undefined;
159
160         console.assert(this._firstFailedStep);
161         return this._firstFailedStep.logs;
162     },
163
164     get previousProductiveIteration()
165     {
166         for (var i = 0; i < this.queue.iterations.length - 1; ++i) {
167             if (this.queue.iterations[i] === this) {
168                 while (++i < this.queue.iterations.length) {
169                     var iteration = this.queue.iterations[i];
170                     if (iteration.productive)
171                         return iteration;
172                 }
173                 break;
174             }
175         }
176         return null;
177     },
178
179     _parseData: function(data)
180     {
181         function collectTestResults(data, stepName)
182         {
183             var testStep = data.steps.findFirst(function(step) { return step.name === stepName; });
184             if (!testStep)
185                 return null;
186
187             var testResults = {};
188
189             if (!testStep.isFinished) {
190                 // The step never even ran, or hasn't finished running.
191                 testResults.finished = false;
192                 return testResults;
193             }
194
195             testResults.finished = true;
196
197             if (!testStep.results || !testStep.results[0]) {
198                 // All tests passed.
199                 testResults.allPassed = true;
200                 return testResults;
201             }
202
203             if (/Exiting early/.test(testStep.results[1][0]))
204                 testResults.tooManyFailures = true;
205
206             function resultSummarizer(matchString, sum, outputLine)
207             {
208                 var match = /^(\d+)\s/.exec(outputLine);
209                 if (!match)
210                     return sum;
211                 if (!outputLine.contains(matchString))
212                     return sum;
213                 if (!sum || sum === -1)
214                     sum = 0;
215                 return sum + parseInt(match[1], 10);
216             }
217
218             testResults.failureCount = testStep.results[1].reduce(resultSummarizer.bind(null, "fail"), undefined);
219             testResults.flakeyCount = testStep.results[1].reduce(resultSummarizer.bind(null, "flake"), undefined);
220             testResults.totalLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "total leak"), undefined);
221             testResults.uniqueLeakCount = testStep.results[1].reduce(resultSummarizer.bind(null, "unique leak"), undefined);
222             testResults.newPassesCount = testStep.results[1].reduce(resultSummarizer.bind(null, "new pass"), undefined);
223             testResults.missingCount = testStep.results[1].reduce(resultSummarizer.bind(null, "missing"), undefined);
224
225             if (!testResults.failureCount && !testResults.flakyCount && !testResults.totalLeakCount && !testResults.uniqueLeakCount && !testResults.newPassesCount && !testResults.missingCount) {
226                 // This step exited with a non-zero exit status, but we didn't find any output about the number of failed tests.
227                 // Something must have gone wrong (e.g., timed out and was killed by buildbot).
228                 testResults.errorOccurred = true;
229             }
230
231             return testResults;
232         }
233
234         console.assert(!this.id || this.id === data.number);
235         this.id = data.number;
236
237         // The property got_revision may have the following forms:
238         //
239         // ["got_revision",{"Internal":"1357","WebKitOpenSource":"2468"},"Source"]
240         // OR
241         // ["got_revision","2468_1357","Source"]
242         // OR
243         // ["got_revision","2468","Source"]
244         //
245         // When extracting the OpenSource revision from property got_revision we don't need to check whether the
246         // value of got_revision is a dictionary (represents multiple codebases) or a string literal because we
247         // assume that got_revision contains the OpenSource revision. However, it may not have the Internal
248         // revision. Therefore, we only look at got_revision to extract the Internal revision when it's
249         // a dictionary.
250
251         var openSourceRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "got_revision" || property[0] === "revision" || property[0] === "opensource_got_revision"; });
252         this.openSourceRevision = parseRevisionProperty(openSourceRevisionProperty, "WebKit");
253
254         var internalRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "internal_got_revision" || isMultiCodebaseGotRevisionProperty(property); });
255         this.internalRevision = parseRevisionProperty(internalRevisionProperty, "Internal");
256
257         this.startTime = new Date(data.times[0] * 1000);
258         this.endTime = new Date(data.times[1] * 1000);
259
260         var layoutTestResults = collectTestResults.call(this, data, "layout-test");
261         this.layoutTestResults = layoutTestResults ? new BuildbotTestResults(this, layoutTestResults) : null;
262
263         var javascriptTestResults = collectTestResults.call(this, data, "jscore-test");
264         this.javascriptTestResults = javascriptTestResults ? new BuildbotTestResults(this, javascriptTestResults) : null;
265
266         var apiTestResults = collectTestResults.call(this, data, "run-api-tests");
267         this.apiTestResults = apiTestResults ? new BuildbotTestResults(this, apiTestResults) : null;
268
269         var platformAPITestResults = collectTestResults.call(this, data, "API tests");
270         this.platformAPITestResults = platformAPITestResults ? new BuildbotTestResults(this, platformAPITestResults) : null;
271
272         var pythonTestResults = collectTestResults.call(this, data, "webkitpy-test");
273         this.pythonTestResults = pythonTestResults ? new BuildbotTestResults(this, pythonTestResults) : null;
274
275         var perlTestResults = collectTestResults.call(this, data, "webkitperl-test");
276         this.perlTestResults = perlTestResults ? new BuildbotTestResults(this, perlTestResults) : null;
277
278         var bindingTestResults = collectTestResults.call(this, data, "bindings-generation-tests");
279         this.bindingTestResults = bindingTestResults ? new BuildbotTestResults(this, bindingTestResults) : null;
280
281         this.loaded = true;
282
283         this._firstFailedStep = data.steps.findFirst(function(step) { return step.results[0] === BuildbotIteration.FAILURE; });
284
285         console.assert(data.results === null || typeof data.results === "number");
286         this._result = data.results;
287
288         this.text = data.text.join(" ");
289
290         if (!data.currentStep)
291             this.finished = true;
292
293         this._productive = this._finished && this._result !== BuildbotIteration.EXCEPTION && this._result !== BuildbotIteration.RETRY;
294         if (this._productive) {
295             var finishedAnyProductiveStep = false;
296             for (var i = 0; i < data.steps.length; ++i) {
297                 var step = data.steps[i];
298                 if (!step.isFinished)
299                     break;
300                 if (step.name in BuildbotIteration.ProductiveSteps) {
301                     finishedAnyProductiveStep = true;
302                     break;
303                 }
304             }
305             this._productive = finishedAnyProductiveStep;
306         }
307
308         // Update the sorting since it is based on the revisions we just loaded.
309         this.queue.sortIterations();
310     },
311
312     _updateWithData: function(data)
313     {
314         if (this.loaded && this._finished)
315             return;
316
317         this._parseData(data);
318         this.dispatchEventToListeners(BuildbotIteration.Event.Updated);
319     },
320
321     update: function()
322     {
323         if (this.loaded && this._finished)
324             return;
325
326         if (this.queue.buildbot.needsAuthentication && this.queue.buildbot.authenticationStatus === Buildbot.AuthenticationStatus.InvalidCredentials)
327             return;
328
329         if (this.isLoading)
330             return;
331
332         this.isLoading = true;
333
334         JSON.load(this.queue.baseURL + "/builds/" + this.id, function(data) {
335             this.isLoading = false;
336             this.queue.buildbot.isAuthenticated = true;
337             if (!data || !data.properties)
338                 return;
339
340             this._updateWithData(data);
341         }.bind(this),
342         function(data) {
343             this.isLoading = false;
344             if (data.errorType === JSON.LoadError && data.errorHTTPCode === 401) {
345                 this.queue.buildbot.isAuthenticated = false;
346                 this.dispatchEventToListeners(BuildbotIteration.Event.UnauthorizedAccess);
347             }
348         }.bind(this), {withCredentials: this.queue.buildbot.needsAuthentication});
349     },
350
351     loadLayoutTestResults: function(callback)
352     {
353         if (this.queue.buildbot.needsAuthentication && this.queue.buildbot.authenticationStatus === Buildbot.AuthenticationStatus.InvalidCredentials)
354             return;
355
356         function collectResults(subtree, predicate)
357         {
358             // Results object is a trie:
359             // directory
360             //   subdirectory
361             //     test1.html
362             //       expected:"PASS"
363             //       actual: "IMAGE"
364             //       report: "REGRESSION"
365             //     test2.html
366             //       expected:"FAIL"
367             //       actual:"TEXT"
368
369             var result = [];
370             for (var key in subtree) {
371                 var value = subtree[key];
372                 console.assert(typeof value === "object");
373                 var isIndividualTest = value.hasOwnProperty("actual") && value.hasOwnProperty("expected");
374                 if (isIndividualTest) {
375                     // Possible values for actual and expected keys: PASS, FAIL, AUDIO, IMAGE, TEXT, IMAGE+TEXT, TIMEOUT, CRASH, MISSING.
376                     // Both actual and expected can be space separated lists. Actual contains two values when retrying a failed test
377                     // gives a different result (retrying may be disabled in tester configuration).
378                     // Possible values for report key (when present): REGRESSION, MISSING, FLAKY.
379
380                     if (predicate(value)) {
381                         var item = {path: key};
382
383                         // FIXME (bug 127186): Crash log URL will be incorrect if crash only happened on retry (e.g. "TEXT CRASH").
384                         // It should point to retries subdirectory, but the information about which attempt failed gets lost here.
385                         if (value.actual.contains("CRASH"))
386                             item.crash = true;
387                         if (value.actual.contains("TIMEOUT"))
388                             item.timeout = true;
389
390                         // FIXME (bug 127186): Similarly, we don't have a good way to present results for something like "TIMEOUT TEXT",
391                         // not even UI wise. For now, only show a diff link if the first attempt has the diff.
392                         if (value.actual.split(" ")[0].contains("TEXT"))
393                             item.has_diff = true;
394
395                         // FIXME (bug 127186): It is particularly unfortunate for image diffs, because we currently only check image results
396                         // on retry (except for reftests), so many times, you will see images on buidbot page, but not on the dashboard.
397                         // FIXME: Find a way to display expected mismatch reftest failures. 
398                         if (value.actual.split(" ")[0].contains("IMAGE") && value.reftest_type != "!=")
399                             item.has_image_diff = true;
400
401                         if (value.has_stderr)
402                             item.has_stderr = true;
403
404                         result.push(item);
405                     }
406
407                 } else {
408                     var nestedTests = collectResults(value, predicate);
409                     for (var i = 0, end = nestedTests.length; i < end; ++i)
410                         nestedTests[i].path = key + "/" + nestedTests[i].path;
411                     result = result.concat(nestedTests);
412                 }
413             }
414
415             return result;
416         }
417
418         JSON.load(this.queue.buildbot.layoutTestFullResultsURLForIteration(this), function(data) {
419             this.queue.buildbot.isAuthenticated = true;
420             this.hasPrettyPatch = data.has_pretty_patch;
421
422             this.layoutTestResults.regressions = collectResults(data.tests, function(info) { return info["report"] === "REGRESSION" });
423             console.assert(data.num_regressions === this.layoutTestResults.regressions.length);
424
425             this.layoutTestResults.flakyTests = collectResults(data.tests, function(info) { return info["report"] === "FLAKY" });
426             console.assert(data.num_flaky === this.layoutTestResults.flakyTests.length);
427
428             this.layoutTestResults.testsWithMissingResults = collectResults(data.tests, function(info) { return info["report"] === "MISSING" });
429             console.assert(data.num_missing === this.layoutTestResults.testsWithMissingResults.length);
430
431             callback();
432         }.bind(this),
433         function(data) {
434             if (data.errorType === JSON.LoadError && data.errorHTTPCode === 401) {
435                 this.queue.buildbot.isAuthenticated = false;
436                 this.dispatchEventToListeners(BuildbotIteration.Event.UnauthorizedAccess);
437             }
438             console.log(data.error);
439             callback();
440         }.bind(this), {jsonpCallbackName: "ADD_RESULTS", withCredentials: this.queue.buildbot.needsAuthentication});
441     }
442 };