70566626361dacb5c5a933de5ca5186829fa9a6d
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / servers / data / gardeningserver / results.js
1 var results = results || {};
2
3 (function() {
4
5 var kTestResultsServer = 'http://test-results.appspot.com/';
6 var kTestResultsQuery = kTestResultsServer + 'testfile?'
7 var kTestType = 'layout-tests';
8 var kResultsName = 'full_results.json';
9 var kMasterName = 'ChromiumWebkit';
10
11 var kLayoutTestResultsServer = 'http://build.chromium.org/f/chromium/layout_test_results/';
12 var kLayoutTestResultsPath = '/results/layout-test-results/';
13
14 var PASS = 'PASS';
15 var TIMEOUT = 'TIMEOUT';
16 var TEXT = 'TEXT';
17 var CRASH = 'CRASH';
18 var IMAGE = 'IMAGE';
19 var IMAGE_TEXT = 'IMAGE+TEXT';
20
21 var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
22
23 var kExpectedImageSuffix = '-expected.png';
24 var kActualImageSuffix = '-actual.png';
25 var kImageDiffSuffix = '-diff.png';
26 var kTextDiffSuffix = '-diff.txt';
27 var kCrashLogSuffix = '-crash-log.txt';
28
29 var kPreferredSuffixOrder = [
30     kExpectedImageSuffix,
31     kActualImageSuffix,
32     kImageDiffSuffix,
33     kTextDiffSuffix,
34     kCrashLogSuffix,
35     // FIXME: Add support for the rest of the result types.
36 ];
37
38 // Kinds of results.
39 results.kActualKind = 'actual';
40 results.kExpectedKind = 'expected';
41 results.kDiffKind = 'diff';
42 results.kUnknownKind = 'unknown';
43
44 // Types of tests.
45 results.kImageType = 'image'
46 results.kTextType = 'text'
47 // FIXME: There are more types of tests.
48
49 function isFailure(result)
50 {
51     return kFailingResults.indexOf(result) != -1;
52 }
53
54 function isSuccess(result)
55 {
56     return result === PASS;
57 }
58
59 function resultsParameters(builderName, testName)
60 {
61     return {
62         builder: builderName,
63         master: kMasterName,
64         testtype: kTestType,
65         name: testName,
66     };
67 }
68
69 function possibleSuffixListFor(failureTypeList)
70 {
71     var suffixList = [];
72
73     function pushImageSuffixes()
74     {
75         suffixList.push(kExpectedImageSuffix);
76         suffixList.push(kActualImageSuffix);
77         suffixList.push(kImageDiffSuffix);
78     }
79
80     function pushTextSuffixes()
81     {
82         // '-expected.txt',
83         // '-actual.txt',
84         suffixList.push(kTextDiffSuffix);
85         // '-wdiff.html',
86         // '-pretty-diff.html',
87     }
88
89     $.each(failureTypeList, function(index, failureType) {
90         switch(failureType) {
91             case IMAGE:
92                 pushImageSuffixes();
93                 break;
94             case TEXT:
95                 pushTextSuffixes();
96                 break;
97             case IMAGE_TEXT:
98                 pushImageSuffixes();
99                 pushTextSuffixes();
100                 break;
101             case CRASH:
102                 suffixList.push(kCrashLogSuffix);
103                 break;
104             default:
105                 // FIXME: Add support for the rest of the result types.
106                 // '-expected.html',
107                 // '-expected-mismatch.html',
108                 // '-expected.wav',
109                 // '-actual.wav',
110                 // ... and possibly more.
111                 break;
112         }
113     });
114
115     return suffixList;
116 }
117
118 function resultsSummaryURL(builderName, testName)
119 {
120     return kTestResultsQuery + $.param(resultsParameters(builderName, testName));
121 }
122
123 function directoryOfResultsSummaryURL(builderName, testName)
124 {
125     var parameters = resultsParameters(builderName, testName);
126     parameters['dir'] = 1;
127     return kTestResultsQuery + $.param(parameters);
128 }
129
130 function ResultsCache()
131 {
132     this._cache = {};
133 }
134
135 ResultsCache.prototype._fetch = function(key, callback)
136 {
137     var self = this;
138
139     var url = kTestResultsServer + 'testfile?key=' + key;
140     base.jsonp(url, function (resultsTree) {
141         self._cache[key] = resultsTree;
142         callback(resultsTree);
143     });
144 };
145
146 // Warning! This function can call callback either synchronously or asynchronously.
147 // FIXME: Consider using setTimeout to make this method always asynchronous.
148 ResultsCache.prototype.get = function(key, callback)
149 {
150     if (key in this._cache) {
151         callback(this._cache[key]);
152         return;
153     }
154     this._fetch(key, callback);
155 };
156
157 var g_resultsCache = new ResultsCache();
158
159 function anyIsFailure(resultsList)
160 {
161     return $.grep(resultsList, isFailure).length > 0;
162 }
163
164 function anyIsSuccess(resultsList)
165 {
166     return $.grep(resultsList, isSuccess).length > 0;
167 }
168
169 function addImpliedExpectations(resultsList)
170 {
171     if (resultsList.indexOf('FAIL') == -1)
172         return resultsList;
173     return resultsList.concat(kFailingResults);
174 }
175
176 function unexpectedResults(resultNode)
177 {
178     var actualResults = resultNode.actual.split(' ');
179     var expectedResults = addImpliedExpectations(resultNode.expected.split(' '))
180
181     return $.grep(actualResults, function(result) {
182         return expectedResults.indexOf(result) == -1;
183     });
184 }
185
186 function isUnexpectedFailure(resultNode)
187 {
188     if (!resultNode)
189         return false;
190     if (anyIsSuccess(resultNode.actual.split(' ')))
191         return false;
192     return anyIsFailure(unexpectedResults(resultNode));
193 }
194
195 function isResultNode(node)
196 {
197     return !!node.actual;
198 }
199
200 results.BuilderResults = function(resultsJSON)
201 {
202     this._resultsJSON = resultsJSON;
203 };
204
205 results.BuilderResults.prototype.unexpectedFailures = function()
206 {
207     return base.filterTree(this._resultsJSON.tests, isResultNode, isUnexpectedFailure);
208 };
209
210 results.unexpectedFailuresByTest = function(resultsByBuilder)
211 {
212     var unexpectedFailures = {};
213
214     $.each(resultsByBuilder, function(builderName, builderResults) {
215         $.each(builderResults.unexpectedFailures(), function(testName, resultNode) {
216             unexpectedFailures[testName] = unexpectedFailures[testName] || {};
217             unexpectedFailures[testName][builderName] = resultNode;
218         });
219     });
220
221     return unexpectedFailures;
222 };
223
224 results.collectUnexpectedResults = function(dictionaryOfResultNodes)
225 {
226     var collectedResults = {};
227     var results = [];
228     $.each(dictionaryOfResultNodes, function(key, resultNode) {
229         results = results.concat(unexpectedResults(resultNode));
230     });
231     return base.uniquifyArray(results);
232 };
233
234 function walkHistory(builderName, testName, callback)
235 {
236     var indexOfNextKeyToFetch = 0;
237     var keyList = [];
238
239     function continueWalk()
240     {
241         if (indexOfNextKeyToFetch >= keyList.length) {
242             processResultNode(0, null);
243             return;
244         }
245
246         var key = keyList[indexOfNextKeyToFetch];
247         ++indexOfNextKeyToFetch;
248         g_resultsCache.get(key, function(resultsTree) {
249             var resultNode = results.resultNodeForTest(resultsTree, testName);
250             var revision = parseInt(resultsTree['revision'])
251             if (isNaN(revision))
252                 revision = 0;
253             processResultNode(revision, resultNode);
254         });
255     }
256
257     function processResultNode(revision, resultNode)
258     {
259         var shouldContinue = callback(revision, resultNode);
260         if (!shouldContinue)
261             return;
262         continueWalk();
263     }
264
265     base.jsonp(directoryOfResultsSummaryURL(builderName, kResultsName), function(directory) {
266         keyList = directory.map(function (element) { return element.key; });
267         continueWalk();
268     });
269 }
270
271 results.regressionRangeForFailure = function(builderName, testName, callback)
272 {
273     var oldestFailingRevision = 0;
274     var newestPassingRevision = 0;
275
276     walkHistory(builderName, testName, function(revision, resultNode) {
277         if (!revision) {
278             callback(oldestFailingRevision, newestPassingRevision);
279             return false;
280         }
281         if (!resultNode) {
282             newestPassingRevision = revision;
283             callback(oldestFailingRevision, newestPassingRevision);
284             return false;
285         }
286         if (isUnexpectedFailure(resultNode)) {
287             oldestFailingRevision = revision;
288             return true;
289         }
290         if (!oldestFailingRevision)
291             return true;  // We need to keep looking for a failing revision.
292         newestPassingRevision = revision;
293         callback(oldestFailingRevision, newestPassingRevision);
294         return false;
295     });
296 };
297
298 function mergeRegressionRanges(regressionRanges)
299 {
300     var mergedRange = {};
301
302     mergedRange.oldestFailingRevision = 0;
303     mergedRange.newestPassingRevision = 0;
304
305     $.each(regressionRanges, function(builderName, range) {
306         if (!range.oldestFailingRevision || !range.newestPassingRevision)
307             return
308
309         if (!mergedRange.oldestFailingRevision)
310             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
311         if (!mergedRange.newestPassingRevision)
312             mergedRange.newestPassingRevision = range.newestPassingRevision;
313
314         if (range.oldestFailingRevision < mergedRange.oldestFailingRevision)
315             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
316         if (range.newestPassingRevision > mergedRange.newestPassingRevision)
317             mergedRange.newestPassingRevision = range.newestPassingRevision;
318     });
319     return mergedRange;
320 }
321
322 results.unifyRegressionRanges = function(builderNameList, testName, callback)
323 {
324     var queriesInFlight = builderNameList.length;
325     if (!queriesInFlight)
326         callback(0, 0);
327
328     var regressionRanges = {};
329     $.each(builderNameList, function(index, builderName) {
330         results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
331             var range = {};
332             range.oldestFailingRevision = oldestFailingRevision;
333             range.newestPassingRevision = newestPassingRevision;
334             regressionRanges[builderName] = range;
335
336             --queriesInFlight;
337             if (!queriesInFlight) {
338                 var mergedRange = mergeRegressionRanges(regressionRanges);
339                 callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
340             }
341         });
342     });
343 };
344
345 results.countFailureOccurances = function(builderNameList, testName, callback)
346 {
347     var queriesInFlight = builderNameList.length;
348     if (!queriesInFlight)
349         callback(0);
350
351     var failureCount = 0;
352     $.each(builderNameList, function(index, builderName) {
353         walkHistory(builderName, testName, function(revision, resultNode) {
354             if (isUnexpectedFailure(resultNode)) {
355                 ++failureCount;
356                 return true;
357             }
358
359             --queriesInFlight;
360             if (!queriesInFlight)
361                 callback(failureCount);
362             return false;
363         });
364     });
365 };
366
367 results.resultNodeForTest = function(resultsTree, testName)
368 {
369     var testNamePath = testName.split('/');
370     var currentNode = resultsTree['tests'];
371     $.each(testNamePath, function(index, segmentName) {
372         if (!currentNode)
373             return;
374         currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
375     });
376     return currentNode;
377 };
378
379 function resultsDirectoryForBuilder(builderName)
380 {
381     return builderName.replace(/[ .()]/g, '_');
382 }
383
384 function resultsDirectoryURL(builderName)
385 {
386     return kLayoutTestResultsServer + resultsDirectoryForBuilder(builderName) + kLayoutTestResultsPath;
387 }
388
389 results.resultKind = function(url)
390 {
391     if (/-actual\.[a-z]+$/.test(url))
392         return results.kActualKind;
393     else if (/-expected\.[a-z]+$/.test(url))
394         return results.kExpectedKind;
395     else if (/diff\.[a-z]+$/.test(url))
396         return results.kDiffKind;
397     return results.kUnknownKind;
398 }
399
400 results.resultType = function(url)
401 {
402     if (/\.png$/.test(url))
403         return results.kImageType;
404     return results.kTextType;
405 }
406
407 function sortResultURLsBySuffix(urls)
408 {
409     var sortedURLs = [];
410     $.each(kPreferredSuffixOrder, function(i, suffix) {
411         $.each(urls, function(j, url) {
412             if (!base.endsWith(url, suffix))
413                 return;
414             sortedURLs.push(url);
415         });
416     });
417     if (sortedURLs.length != urls.length)
418         throw "sortResultURLsBySuffix failed to return the same number of URLs."
419     return sortedURLs;
420 }
421
422 results.fetchResultsURLs = function(builderName, testName, failureTypeList, callback)
423 {
424     var stem = resultsDirectoryURL(builderName);
425     var testNameStem = base.trimExtension(testName);
426
427     var suffixList = possibleSuffixListFor(failureTypeList);
428
429     var resultURLs = [];
430     var requestsInFlight = suffixList.length;
431
432     if (!requestsInFlight) {
433         callback([]);
434         return;
435     }
436
437     function checkComplete()
438     {
439         if (--requestsInFlight == 0)
440             callback(sortResultURLsBySuffix(resultURLs));
441     }
442
443     $.each(suffixList, function(index, suffix) {
444         var url = stem + testNameStem + suffix;
445         base.probe(url, {
446             success: function() {
447                 resultURLs.push(url);
448                 checkComplete();
449             },
450             error: checkComplete,
451         });
452     });
453 };
454
455 results.fetchResultsForBuilder = function(builderName, onsuccess)
456 {
457     base.jsonp(resultsSummaryURL(builderName, kResultsName), function(resultsTree) {
458         onsuccess(new results.BuilderResults(resultsTree));
459     });
460 };
461
462 results.fetchResultsByBuilder = function(builderNameList, onsuccess)
463 {
464     var resultsByBuilder = {}
465     var requestsInFlight = builderNameList.length;
466     $.each(builderNameList, function(index, builderName) {
467         results.fetchResultsForBuilder(builderName, function(resultsTree) {
468             resultsByBuilder[builderName] = resultsTree;
469             --requestsInFlight;
470             if (!requestsInFlight)
471                 onsuccess(resultsByBuilder);
472         });
473     });
474 };
475
476 })();