garden-o-matic should have a "rebaseline" button
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / servers / data / gardeningserver / results_unittests.js
1 (function () {
2
3 module("results");
4
5 var kExampleResultsJSON = {
6     "tests": {
7         "scrollbars": {
8             "custom-scrollbar-with-incomplete-style.html": {
9                 "expected": "IMAGE",
10                 "actual": "IMAGE"
11             },
12             "flaky-scrollbar.html": {
13                 "expected": "PASS",
14                 "actual": "PASS TEXT"
15             }
16         },
17         "userscripts": {
18             "user-script-video-document.html": {
19                 "expected": "FAIL",
20                 "actual": "TEXT"
21             },
22             "another-test.html": {
23                 "expected": "PASS",
24                 "actual": "TEXT"
25             }
26         },
27     },
28     "skipped": 339,
29     "num_regressions": 14,
30     "interrupted": false,
31     "layout_tests_dir": "\/mnt\/data\/b\/build\/slave\/Webkit_Linux\/build\/src\/third_party\/WebKit\/LayoutTests",
32     "version": 3,
33     "num_passes": 15566,
34     "has_pretty_patch": false,
35     "fixable": 1233,
36     "num_flaky":1,
37     "uses_expectations_file": true,
38     "has_wdiff": true,
39     "revision": "90430"
40 };
41
42 test("BuilderResults.unexpectedFailures", 1, function() {
43     var builderResults = new results.BuilderResults(kExampleResultsJSON);
44     var unexpectedFailures = builderResults.unexpectedFailures();
45     deepEqual(unexpectedFailures, {
46         "userscripts/another-test.html": {
47             "expected": "PASS",
48             "actual": "TEXT"
49         }
50     });
51 });
52
53 test("unexpectedFailuresByTest", 1, function() {
54     var builderResults = new results.BuilderResults(kExampleResultsJSON);
55     var unexpectedFailuresByTest = results.unexpectedFailuresByTest({
56         "Mock Builder": builderResults
57     });
58     deepEqual(unexpectedFailuresByTest, {
59         "userscripts/another-test.html": {
60             "Mock Builder": {
61                 "expected": "PASS",
62                 "actual": "TEXT"
63             }
64         }
65     });
66 });
67
68 test("resultKind", 12, function() {
69     equals(results.resultKind("http://example.com/foo-actual.txt"), "actual");
70     equals(results.resultKind("http://example.com/foo-expected.txt"), "expected");
71     equals(results.resultKind("http://example.com/foo-diff.txt"), "diff");
72     equals(results.resultKind("http://example.com/foo.bar-actual.txt"), "actual");
73     equals(results.resultKind("http://example.com/foo.bar-expected.txt"), "expected");
74     equals(results.resultKind("http://example.com/foo.bar-diff.txt"), "diff");
75     equals(results.resultKind("http://example.com/foo-actual.png"), "actual");
76     equals(results.resultKind("http://example.com/foo-expected.png"), "expected");
77     equals(results.resultKind("http://example.com/foo-diff.png"), "diff");
78     equals(results.resultKind("http://example.com/foo-pretty-diff.html"), "diff");
79     equals(results.resultKind("http://example.com/foo-wdiff.html"), "diff");
80     equals(results.resultKind("http://example.com/foo-xyz.html"), "unknown");
81 });
82
83 test("resultType", 12, function() {
84     equals(results.resultType("http://example.com/foo-actual.txt"), "text");
85     equals(results.resultType("http://example.com/foo-expected.txt"), "text");
86     equals(results.resultType("http://example.com/foo-diff.txt"), "text");
87     equals(results.resultType("http://example.com/foo.bar-actual.txt"), "text");
88     equals(results.resultType("http://example.com/foo.bar-expected.txt"), "text");
89     equals(results.resultType("http://example.com/foo.bar-diff.txt"), "text");
90     equals(results.resultType("http://example.com/foo-actual.png"), "image");
91     equals(results.resultType("http://example.com/foo-expected.png"), "image");
92     equals(results.resultType("http://example.com/foo-diff.png"), "image");
93     equals(results.resultType("http://example.com/foo-pretty-diff.html"), "text");
94     equals(results.resultType("http://example.com/foo-wdiff.html"), "text");
95     equals(results.resultType("http://example.com/foo.xyz"), "text");
96 });
97
98 test("resultNodeForTest", 4, function() {
99     deepEqual(results.resultNodeForTest(kExampleResultsJSON, "userscripts/another-test.html"), {
100         "expected": "PASS",
101         "actual": "TEXT"
102     });
103     equals(results.resultNodeForTest(kExampleResultsJSON, "foo.html"), null);
104     equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo.html"), null);
105     equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo/bar.html"), null);
106 });
107
108 function NetworkSimulator()
109 {
110     this._pendingCallbacks = [];
111 };
112
113 NetworkSimulator.prototype.scheduleCallback = function(callback)
114 {
115     this._pendingCallbacks.push(callback);
116 }
117
118 NetworkSimulator.prototype.runTest = function(testCase)
119 {
120     var self = this;
121     var realBase = window.base;
122
123     window.base = {};
124     base.endsWith = realBase.endsWith;
125     base.trimExtension = realBase.trimExtension;
126     if (self.probeHook)
127         base.probe = self.probeHook;
128     if (self.jsonpHook)
129         base.jsonp = self.jsonpHook;
130
131     testCase();
132
133     while (this._pendingCallbacks.length) {
134         var callback = this._pendingCallbacks.shift();
135         callback();
136     }
137
138     window.base = realBase;
139     equal(window.base, realBase, "Failed to restore real base!");
140 }
141
142 test("walkHistory", 6, function() {
143     var simulator = new NetworkSimulator();
144
145     var keyMap = {
146         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw": {
147             "tests": {
148                 "userscripts": {
149                     "another-test.html": {
150                         "expected": "PASS",
151                         "actual": "TEXT"
152                     }
153                 },
154             },
155             "revision": "90430"
156         },
157         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw":{
158             "tests": {
159                 "userscripts": {
160                     "user-script-video-document.html": {
161                         "expected": "FAIL",
162                         "actual": "TEXT"
163                     },
164                     "another-test.html": {
165                         "expected": "PASS",
166                         "actual": "TEXT"
167                     }
168                 },
169             },
170             "revision": "90429"
171         },
172         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw":{
173             "tests": {
174                 "userscripts": {
175                     "another-test.html": {
176                         "expected": "PASS",
177                         "actual": "TEXT"
178                     }
179                 },
180             },
181             "revision": "90426"
182         },
183         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw":{
184             "tests": {
185                 "userscripts": {
186                     "user-script-video-document.html": {
187                         "expected": "FAIL",
188                         "actual": "TEXT"
189                     },
190                 },
191             },
192             "revision": "90424"
193         },
194         "abc":{
195             "tests": {
196                 "userscripts": {
197                     "another-test.html": {
198                         "expected": "PASS",
199                         "actual": "TEXT"
200                     }
201                 },
202             },
203             "revision": "90426"
204         },
205         "xyz":{
206             "tests": {
207             },
208             "revision": "90425"
209         }
210     };
211
212     simulator.jsonpHook = function(url, callback) {
213         simulator.scheduleCallback(function() {
214             if (/dir=1/.test(url)) {
215                 if (/builder=Mock/.test(url)) {
216                     callback([
217                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
218                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
219                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
220                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
221                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
222                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
223                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
224                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
225                     ]);
226                 } else if (/builder=Another/.test(url)) {
227                     callback([
228                         { "key": "abc" },
229                         { "key": "xyz" },
230                     ]);
231                 } else {
232                     ok(false, 'Unexpected URL: ' + url);
233                 }
234             } else {
235                 var key = url.match(/key=([^&]+)/)[1];
236                 callback(keyMap[key]);
237             }
238         });
239     };
240     simulator.runTest(function() {
241         results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
242             equals(oldestFailingRevision, 90426);
243             equals(newestPassingRevision, 90424);
244         });
245
246         results.unifyRegressionRanges(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
247             equals(oldestFailingRevision, 90426);
248             equals(newestPassingRevision, 90425);
249         });
250
251         results.countFailureOccurances(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(failureCount) {
252             equals(failureCount, 4);
253         });
254     });
255 });
256
257 test("walkHistory (no revision)", 3, function() {
258     var simulator = new NetworkSimulator();
259
260     var keyMap = {
261         "vsfdsfdsafsdafasd": {
262             "tests": {
263                 "userscripts": {
264                     "another-test.html": {
265                         "expected": "PASS",
266                         "actual": "TEXT"
267                     }
268                 },
269             },
270             "revision": ""
271         },
272         "gavsavsrfgwaevwefawvae":{
273             "tests": {
274             },
275             "revision": ""
276         },
277     };
278
279     simulator.jsonpHook = function(url, callback) {
280         simulator.scheduleCallback(function() {
281             if (/dir=1/.test(url)) {
282                 callback([
283                     { "key": "vsfdsfdsafsdafasd" },
284                     { "key": "gavsavsrfgwaevwefawvae" },
285                 ]);
286             } else {
287                 var key = url.match(/key=([^&]+)/)[1];
288                 callback(keyMap[key]);
289             }
290         });
291     };
292
293     simulator.runTest(function() {
294         results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
295             equals(oldestFailingRevision, 0);
296             equals(newestPassingRevision, 0);
297         });
298     });
299 });
300
301 test("collectUnexpectedResults", 1, function() {
302     var dictionaryOfResultNodes = {
303         "foo": {
304             "expected": "IMAGE",
305             "actual": "IMAGE"
306         },
307         "bar": {
308             "expected": "PASS",
309             "actual": "PASS TEXT"
310         },
311         "baz": {
312             "expected": "TEXT",
313             "actual": "IMAGE"
314         },
315         "qux": {
316             "expected": "PASS",
317             "actual": "TEXT"
318         },
319         "taco": {
320             "expected": "PASS",
321             "actual": "TEXT"
322         },
323     };
324
325     var collectedResults = results.collectUnexpectedResults(dictionaryOfResultNodes);
326     deepEqual(collectedResults, ["TEXT", "IMAGE"]);
327 });
328
329 test("failureTypeToExtensionList", 5, function() {
330     deepEqual(results.failureTypeToExtensionList('TEXT'), ['txt']);
331     deepEqual(results.failureTypeToExtensionList('IMAGE+TEXT'), ['txt', 'png']);
332     deepEqual(results.failureTypeToExtensionList('IMAGE'), ['png']);
333     deepEqual(results.failureTypeToExtensionList('CRASH'), []);
334     deepEqual(results.failureTypeToExtensionList('TIMEOUT'), []);
335 });
336
337 test("canRebaseline", 6, function() {
338     ok(results.canRebaseline(['TEXT']));
339     ok(results.canRebaseline(['IMAGE+TEXT', 'CRASH']));
340     ok(results.canRebaseline(['IMAGE']));
341     ok(!results.canRebaseline(['CRASH']));
342     ok(!results.canRebaseline(['TIMEOUT']));
343     ok(!results.canRebaseline([]));
344 });
345
346 test("fetchResultsURLs", 4, function() {
347     var simulator = new NetworkSimulator();
348
349     var probedURLs = [];
350     simulator.probeHook = function(url, options)
351     {
352         simulator.scheduleCallback(function() {
353             probedURLs.push(url);
354             if (base.endsWith(url, '.txt'))
355                 options.success.call();
356             else
357                 options.error.call();
358         });
359     };
360
361     simulator.runTest(function() {
362         results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", ['IMAGE', 'CRASH'], function(resultURLs) {
363             deepEqual(resultURLs, [
364                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-crash-log.txt"
365             ]);
366         });
367         results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", ['TIMEOUT'], function(resultURLs) {
368             deepEqual(resultURLs, []);
369         });
370     });
371
372     deepEqual(probedURLs, [
373         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.png",
374         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.png",
375         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.png",
376         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-crash-log.txt"
377     ]);
378 });
379
380 })();