garden-o-matic should have a "rebaseline" button
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / servers / data / gardeningserver / results.js
1 var results = results || {};
2
3 (function() {
4
5 var kTestResultsServer = 'http://test-results.appspot.com/';
6 var kTestResultsQuery = kTestResultsServer + 'testfile?'
7 var kTestType = 'layout-tests';
8 var kResultsName = 'full_results.json';
9 var kMasterName = 'ChromiumWebkit';
10
11 var kLayoutTestResultsServer = 'http://build.chromium.org/f/chromium/layout_test_results/';
12 var kLayoutTestResultsPath = '/results/layout-test-results/';
13
14 var PASS = 'PASS';
15 var TIMEOUT = 'TIMEOUT';
16 var TEXT = 'TEXT';
17 var CRASH = 'CRASH';
18 var IMAGE = 'IMAGE';
19 var IMAGE_TEXT = 'IMAGE+TEXT';
20
21 var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
22
23 var kExpectedImageSuffix = '-expected.png';
24 var kActualImageSuffix = '-actual.png';
25 var kImageDiffSuffix = '-diff.png';
26 var kTextDiffSuffix = '-diff.txt';
27 var kCrashLogSuffix = '-crash-log.txt';
28
29 var kPNGExtension = 'png';
30 var kTXTExtension = 'txt';
31
32 var kPreferredSuffixOrder = [
33     kExpectedImageSuffix,
34     kActualImageSuffix,
35     kImageDiffSuffix,
36     kTextDiffSuffix,
37     kCrashLogSuffix,
38     // FIXME: Add support for the rest of the result types.
39 ];
40
41 // Kinds of results.
42 results.kActualKind = 'actual';
43 results.kExpectedKind = 'expected';
44 results.kDiffKind = 'diff';
45 results.kUnknownKind = 'unknown';
46
47 // Types of tests.
48 results.kImageType = 'image'
49 results.kTextType = 'text'
50 // FIXME: There are more types of tests.
51
52 function isFailure(result)
53 {
54     return kFailingResults.indexOf(result) != -1;
55 }
56
57 function isSuccess(result)
58 {
59     return result === PASS;
60 }
61
62 function resultsParameters(builderName, testName)
63 {
64     return {
65         builder: builderName,
66         master: kMasterName,
67         testtype: kTestType,
68         name: testName,
69     };
70 }
71
72 function possibleSuffixListFor(failureTypeList)
73 {
74     var suffixList = [];
75
76     function pushImageSuffixes()
77     {
78         suffixList.push(kExpectedImageSuffix);
79         suffixList.push(kActualImageSuffix);
80         suffixList.push(kImageDiffSuffix);
81     }
82
83     function pushTextSuffixes()
84     {
85         // '-expected.txt',
86         // '-actual.txt',
87         suffixList.push(kTextDiffSuffix);
88         // '-wdiff.html',
89         // '-pretty-diff.html',
90     }
91
92     $.each(failureTypeList, function(index, failureType) {
93         switch(failureType) {
94         case IMAGE:
95             pushImageSuffixes();
96             break;
97         case TEXT:
98             pushTextSuffixes();
99             break;
100         case IMAGE_TEXT:
101             pushImageSuffixes();
102             pushTextSuffixes();
103             break;
104         case CRASH:
105             suffixList.push(kCrashLogSuffix);
106             break;
107         default:
108             // FIXME: Add support for the rest of the result types.
109             // '-expected.html',
110             // '-expected-mismatch.html',
111             // '-expected.wav',
112             // '-actual.wav',
113             // ... and possibly more.
114             break;
115         }
116     });
117
118     return suffixList;
119 }
120
121 results.failureTypeToExtensionList = function(failureType)
122 {
123     switch(failureType) {
124     case IMAGE:
125         return [kPNGExtension];
126     case TEXT:
127         return [kTXTExtension];
128     case IMAGE_TEXT:
129         return [kTXTExtension, kPNGExtension];
130     default:
131         // FIXME: Add support for the rest of the result types.
132         // '-expected.html',
133         // '-expected-mismatch.html',
134         // '-expected.wav',
135         // '-actual.wav',
136         // ... and possibly more.
137         return [];
138     }
139 };
140
141 results.canRebaseline = function(failureTypeList)
142 {
143     return failureTypeList.some(function(element) {
144         return results.failureTypeToExtensionList(element).length > 0;
145     });
146 };
147
148 function resultsSummaryURL(builderName, testName)
149 {
150     return kTestResultsQuery + $.param(resultsParameters(builderName, testName));
151 }
152
153 function directoryOfResultsSummaryURL(builderName, testName)
154 {
155     var parameters = resultsParameters(builderName, testName);
156     parameters['dir'] = 1;
157     return kTestResultsQuery + $.param(parameters);
158 }
159
160 function ResultsCache()
161 {
162     this._cache = {};
163 }
164
165 ResultsCache.prototype._fetch = function(key, callback)
166 {
167     var self = this;
168
169     var url = kTestResultsServer + 'testfile?key=' + key;
170     base.jsonp(url, function (resultsTree) {
171         self._cache[key] = resultsTree;
172         callback(resultsTree);
173     });
174 };
175
176 // Warning! This function can call callback either synchronously or asynchronously.
177 // FIXME: Consider using setTimeout to make this method always asynchronous.
178 ResultsCache.prototype.get = function(key, callback)
179 {
180     if (key in this._cache) {
181         callback(this._cache[key]);
182         return;
183     }
184     this._fetch(key, callback);
185 };
186
187 var g_resultsCache = new ResultsCache();
188
189 function anyIsFailure(resultsList)
190 {
191     return $.grep(resultsList, isFailure).length > 0;
192 }
193
194 function anyIsSuccess(resultsList)
195 {
196     return $.grep(resultsList, isSuccess).length > 0;
197 }
198
199 function addImpliedExpectations(resultsList)
200 {
201     if (resultsList.indexOf('FAIL') == -1)
202         return resultsList;
203     return resultsList.concat(kFailingResults);
204 }
205
206 function unexpectedResults(resultNode)
207 {
208     var actualResults = resultNode.actual.split(' ');
209     var expectedResults = addImpliedExpectations(resultNode.expected.split(' '))
210
211     return $.grep(actualResults, function(result) {
212         return expectedResults.indexOf(result) == -1;
213     });
214 }
215
216 function isUnexpectedFailure(resultNode)
217 {
218     if (!resultNode)
219         return false;
220     if (anyIsSuccess(resultNode.actual.split(' ')))
221         return false;
222     return anyIsFailure(unexpectedResults(resultNode));
223 }
224
225 function isResultNode(node)
226 {
227     return !!node.actual;
228 }
229
230 results.BuilderResults = function(resultsJSON)
231 {
232     this._resultsJSON = resultsJSON;
233 };
234
235 results.BuilderResults.prototype.unexpectedFailures = function()
236 {
237     return base.filterTree(this._resultsJSON.tests, isResultNode, isUnexpectedFailure);
238 };
239
240 results.unexpectedFailuresByTest = function(resultsByBuilder)
241 {
242     var unexpectedFailures = {};
243
244     $.each(resultsByBuilder, function(builderName, builderResults) {
245         $.each(builderResults.unexpectedFailures(), function(testName, resultNode) {
246             unexpectedFailures[testName] = unexpectedFailures[testName] || {};
247             unexpectedFailures[testName][builderName] = resultNode;
248         });
249     });
250
251     return unexpectedFailures;
252 };
253
254 results.collectUnexpectedResults = function(dictionaryOfResultNodes)
255 {
256     var collectedResults = {};
257     var results = [];
258     $.each(dictionaryOfResultNodes, function(key, resultNode) {
259         results = results.concat(unexpectedResults(resultNode));
260     });
261     return base.uniquifyArray(results);
262 };
263
264 function walkHistory(builderName, testName, callback)
265 {
266     var indexOfNextKeyToFetch = 0;
267     var keyList = [];
268
269     function continueWalk()
270     {
271         if (indexOfNextKeyToFetch >= keyList.length) {
272             processResultNode(0, null);
273             return;
274         }
275
276         var key = keyList[indexOfNextKeyToFetch];
277         ++indexOfNextKeyToFetch;
278         g_resultsCache.get(key, function(resultsTree) {
279             var resultNode = results.resultNodeForTest(resultsTree, testName);
280             var revision = parseInt(resultsTree['revision'])
281             if (isNaN(revision))
282                 revision = 0;
283             processResultNode(revision, resultNode);
284         });
285     }
286
287     function processResultNode(revision, resultNode)
288     {
289         var shouldContinue = callback(revision, resultNode);
290         if (!shouldContinue)
291             return;
292         continueWalk();
293     }
294
295     base.jsonp(directoryOfResultsSummaryURL(builderName, kResultsName), function(directory) {
296         keyList = directory.map(function (element) { return element.key; });
297         continueWalk();
298     });
299 }
300
301 results.regressionRangeForFailure = function(builderName, testName, callback)
302 {
303     var oldestFailingRevision = 0;
304     var newestPassingRevision = 0;
305
306     walkHistory(builderName, testName, function(revision, resultNode) {
307         if (!revision) {
308             callback(oldestFailingRevision, newestPassingRevision);
309             return false;
310         }
311         if (!resultNode) {
312             newestPassingRevision = revision;
313             callback(oldestFailingRevision, newestPassingRevision);
314             return false;
315         }
316         if (isUnexpectedFailure(resultNode)) {
317             oldestFailingRevision = revision;
318             return true;
319         }
320         if (!oldestFailingRevision)
321             return true;  // We need to keep looking for a failing revision.
322         newestPassingRevision = revision;
323         callback(oldestFailingRevision, newestPassingRevision);
324         return false;
325     });
326 };
327
328 function mergeRegressionRanges(regressionRanges)
329 {
330     var mergedRange = {};
331
332     mergedRange.oldestFailingRevision = 0;
333     mergedRange.newestPassingRevision = 0;
334
335     $.each(regressionRanges, function(builderName, range) {
336         if (!range.oldestFailingRevision || !range.newestPassingRevision)
337             return
338
339         if (!mergedRange.oldestFailingRevision)
340             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
341         if (!mergedRange.newestPassingRevision)
342             mergedRange.newestPassingRevision = range.newestPassingRevision;
343
344         if (range.oldestFailingRevision < mergedRange.oldestFailingRevision)
345             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
346         if (range.newestPassingRevision > mergedRange.newestPassingRevision)
347             mergedRange.newestPassingRevision = range.newestPassingRevision;
348     });
349     return mergedRange;
350 }
351
352 results.unifyRegressionRanges = function(builderNameList, testName, callback)
353 {
354     var queriesInFlight = builderNameList.length;
355     if (!queriesInFlight)
356         callback(0, 0);
357
358     var regressionRanges = {};
359     $.each(builderNameList, function(index, builderName) {
360         results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
361             var range = {};
362             range.oldestFailingRevision = oldestFailingRevision;
363             range.newestPassingRevision = newestPassingRevision;
364             regressionRanges[builderName] = range;
365
366             --queriesInFlight;
367             if (!queriesInFlight) {
368                 var mergedRange = mergeRegressionRanges(regressionRanges);
369                 callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
370             }
371         });
372     });
373 };
374
375 results.countFailureOccurances = function(builderNameList, testName, callback)
376 {
377     var queriesInFlight = builderNameList.length;
378     if (!queriesInFlight)
379         callback(0);
380
381     var failureCount = 0;
382     $.each(builderNameList, function(index, builderName) {
383         walkHistory(builderName, testName, function(revision, resultNode) {
384             if (isUnexpectedFailure(resultNode)) {
385                 ++failureCount;
386                 return true;
387             }
388
389             --queriesInFlight;
390             if (!queriesInFlight)
391                 callback(failureCount);
392             return false;
393         });
394     });
395 };
396
397 results.resultNodeForTest = function(resultsTree, testName)
398 {
399     var testNamePath = testName.split('/');
400     var currentNode = resultsTree['tests'];
401     $.each(testNamePath, function(index, segmentName) {
402         if (!currentNode)
403             return;
404         currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
405     });
406     return currentNode;
407 };
408
409 function resultsDirectoryForBuilder(builderName)
410 {
411     return builderName.replace(/[ .()]/g, '_');
412 }
413
414 function resultsDirectoryURL(builderName)
415 {
416     return kLayoutTestResultsServer + resultsDirectoryForBuilder(builderName) + kLayoutTestResultsPath;
417 }
418
419 results.resultKind = function(url)
420 {
421     if (/-actual\.[a-z]+$/.test(url))
422         return results.kActualKind;
423     else if (/-expected\.[a-z]+$/.test(url))
424         return results.kExpectedKind;
425     else if (/diff\.[a-z]+$/.test(url))
426         return results.kDiffKind;
427     return results.kUnknownKind;
428 }
429
430 results.resultType = function(url)
431 {
432     if (/\.png$/.test(url))
433         return results.kImageType;
434     return results.kTextType;
435 }
436
437 function sortResultURLsBySuffix(urls)
438 {
439     var sortedURLs = [];
440     $.each(kPreferredSuffixOrder, function(i, suffix) {
441         $.each(urls, function(j, url) {
442             if (!base.endsWith(url, suffix))
443                 return;
444             sortedURLs.push(url);
445         });
446     });
447     if (sortedURLs.length != urls.length)
448         throw "sortResultURLsBySuffix failed to return the same number of URLs."
449     return sortedURLs;
450 }
451
452 results.fetchResultsURLs = function(builderName, testName, failureTypeList, callback)
453 {
454     var stem = resultsDirectoryURL(builderName);
455     var testNameStem = base.trimExtension(testName);
456
457     var suffixList = possibleSuffixListFor(failureTypeList);
458
459     var resultURLs = [];
460     var requestsInFlight = suffixList.length;
461
462     if (!requestsInFlight) {
463         callback([]);
464         return;
465     }
466
467     function checkComplete()
468     {
469         if (--requestsInFlight == 0)
470             callback(sortResultURLsBySuffix(resultURLs));
471     }
472
473     $.each(suffixList, function(index, suffix) {
474         var url = stem + testNameStem + suffix;
475         base.probe(url, {
476             success: function() {
477                 resultURLs.push(url);
478                 checkComplete();
479             },
480             error: checkComplete,
481         });
482     });
483 };
484
485 results.fetchResultsForBuilder = function(builderName, onsuccess)
486 {
487     base.jsonp(resultsSummaryURL(builderName, kResultsName), function(resultsTree) {
488         onsuccess(new results.BuilderResults(resultsTree));
489     });
490 };
491
492 results.fetchResultsByBuilder = function(builderNameList, onsuccess)
493 {
494     var resultsByBuilder = {}
495     var requestsInFlight = builderNameList.length;
496     $.each(builderNameList, function(index, builderName) {
497         results.fetchResultsForBuilder(builderName, function(resultsTree) {
498             resultsByBuilder[builderName] = resultsTree;
499             --requestsInFlight;
500             if (!requestsInFlight)
501                 onsuccess(resultsByBuilder);
502         });
503     });
504 };
505
506 })();