Teach TestFailures how to load, parse, and interpret NRWT test results
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / public_html / TestFailures / LayoutTestResultsLoader.js
1 /*
2  * Copyright (C) 2011 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 function LayoutTestResultsLoader(builder) {
27     this._builder = builder;
28 }
29
30 LayoutTestResultsLoader.prototype = {
31     start: function(buildName, callback, errorCallback) {
32         var cacheKey = 'LayoutTestResultsLoader.' + this._builder.name + '.' + buildName;
33         const currentCachedDataVersion = 2;
34         if (PersistentCache.contains(cacheKey)) {
35             var cachedData = PersistentCache.get(cacheKey);
36             if (cachedData.version === currentCachedDataVersion) {
37                 callback(cachedData.tests, cachedData.tooManyFailures);
38                 return;
39             }
40         }
41
42         var result = { tests: {}, tooManyFailures: false, version: currentCachedDataVersion };
43
44         function cacheParseResultsAndCallCallback(parseResult) {
45             result.tests = parseResult.tests;
46             result.tooManyFailures = parseResult.tooManyFailures;
47
48             PersistentCache.set(cacheKey, result);
49             callback(result.tests, result.tooManyFailures);
50         }
51
52         var self = this;
53         self._fetchAndParseNRWTResults(buildName, cacheParseResultsAndCallCallback, function() {
54             self._fetchAndParseORWTResults(buildName, cacheParseResultsAndCallCallback, function() {
55                 // We couldn't fetch results for this build.
56                 PersistentCache.set(cacheKey, result);
57                 errorCallback(result.tests, result.tooManyFailures);
58             });
59         });
60     },
61
62     _fetchAndParseNRWTResults: function(buildName, successCallback, errorCallback) {
63         getResource(this._builder.resultsDirectoryURL(buildName) + 'full_results.json', function(xhr) {
64             successCallback((new NRWTResultsParser()).parse(xhr.responseText));
65         },
66         function(xhr) {
67             errorCallback();
68         });
69     },
70
71     _fetchAndParseORWTResults: function(buildName, successCallback, errorCallback) {
72         var parsedBuildName = this._builder.buildbot.parseBuildName(buildName);
73
74         // http://webkit.org/b/62380 was fixed in r89610.
75         var resultsHTMLSupportsTooManyFailuresInfo = parsedBuildName.revision >= 89610;
76
77         var result = { tests: {}, tooManyFailures: false };
78
79         var self = this;
80
81         function fetchAndParseResultsHTMLAndCallCallback() {
82             getResource(self._builder.resultsPageURL(buildName), function(xhr) {
83                 var parseResult = (new ORWTResultsParser()).parse(xhr.responseText);
84                 result.tests = parseResult.tests;
85                 if (resultsHTMLSupportsTooManyFailuresInfo)
86                     result.tooManyFailures = parseResult.tooManyFailures;
87
88                 successCallback(result);
89             },
90             function(xhr) {
91                 // We failed to fetch results.html. run-webkit-tests must have aborted early.
92                 errorCallback();
93             });
94         }
95
96         if (resultsHTMLSupportsTooManyFailuresInfo) {
97             fetchAndParseResultsHTMLAndCallCallback();
98             return;
99         }
100
101         self._builder.getNumberOfFailingTests(parsedBuildName.buildNumber, function(failingTestCount, tooManyFailures) {
102             result.tooManyFailures = tooManyFailures;
103
104             if (failingTestCount < 0) {
105                 // The number of failing tests couldn't be determined.
106                 errorCallback();
107                 return;
108             }
109
110             if (!failingTestCount) {
111                 // All tests passed.
112                 successCallback(result);
113                 return;
114             }
115
116             // Find out which tests failed.
117             fetchAndParseResultsHTMLAndCallCallback();
118         });
119     },
120 };