2011-05-31 Mikhail Naganov <mnaganov@chromium.org>
[WebKit-https.git] / Tools / TestResultServer / model / jsonresults_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 try:
30     import jsonresults
31     from jsonresults import JsonResults
32 except ImportError:
33     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
34     raise
35
36 import unittest
37
38
39 JSON_RESULTS_TEMPLATE = (
40     '{"Webkit":{'
41     '"allFixableCount":[[TESTDATA_COUNT]],'
42     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
43     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
44     '"deferredCounts":[[TESTDATA_COUNTS]],'
45     '"fixableCount":[[TESTDATA_COUNT]],'
46     '"fixableCounts":[[TESTDATA_COUNTS]],'
47     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
48     '"tests":{[TESTDATA_TESTS]},'
49     '"webkitRevision":[[TESTDATA_WEBKITREVISION]],'
50     '"wontfixCounts":[[TESTDATA_COUNTS]]'
51     '},'
52     '"version":[VERSION]'
53     '}')
54
55 JSON_RESULTS_COUNTS_TEMPLATE = (
56     '{'
57     '"C":[TESTDATA],'
58     '"F":[TESTDATA],'
59     '"I":[TESTDATA],'
60     '"O":[TESTDATA],'
61     '"P":[TESTDATA],'
62     '"T":[TESTDATA],'
63     '"X":[TESTDATA],'
64     '"Z":[TESTDATA]}')
65
66 JSON_RESULTS_DIRECTORY_TEMPLATE = '"[TESTDATA_DIRECTORY]":{[TESTDATA_DATA]}'
67
68 JSON_RESULTS_TESTS_TEMPLATE = (
69     '"[TESTDATA_TEST_NAME]":{'
70     '"results":[[TESTDATA_TEST_RESULTS]],'
71     '"times":[[TESTDATA_TEST_TIMES]]}')
72
73 JSON_RESULTS_PREFIX = "ADD_RESULTS("
74 JSON_RESULTS_SUFFIX = ");"
75
76 JSON_RESULTS_TEST_LIST_TEMPLATE = (
77     '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}')
78
79
80 class JsonResultsTest(unittest.TestCase):
81     def setUp(self):
82         self._builder = "Webkit"
83
84     def _make_test_json(self, test_data):
85         if not test_data:
86             return JSON_RESULTS_PREFIX + JSON_RESULTS_SUFFIX
87
88         builds = test_data["builds"]
89         tests = test_data["tests"]
90         if not builds or not tests:
91             return JSON_RESULTS_PREFIX + JSON_RESULTS_SUFFIX
92
93         json = JSON_RESULTS_TEMPLATE
94
95         counts = []
96         build_numbers = []
97         webkit_revision = []
98         chrome_revision = []
99         times = []
100         for build in builds:
101             counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
102             build_numbers.append("1000%s" % build)
103             webkit_revision.append("2000%s" % build)
104             chrome_revision.append("3000%s" % build)
105             times.append("100000%s000" % build)
106
107         json = json.replace("[TESTDATA_COUNTS]", ",".join(counts))
108         json = json.replace("[TESTDATA_COUNT]", ",".join(builds))
109         json = json.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
110         json = json.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
111         json = json.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
112         json = json.replace("[TESTDATA_TIMES]", ",".join(times))
113
114         if "version" in test_data:
115             json = json.replace("[VERSION]", str(test_data["version"]))
116         else:
117             json = json.replace("[VERSION]", "3")
118
119         json_tests = []
120         for (name, test) in sorted(tests.iteritems()):
121             json_tests.append(self._parse_tests_dict(name, test))
122
123         json = json.replace("[TESTDATA_TESTS]", ",".join(json_tests))
124
125         return JSON_RESULTS_PREFIX + json + JSON_RESULTS_SUFFIX
126
127     def _parse_tests_dict(self, name, test):
128         if "results" in test:
129             test_results = JSON_RESULTS_TESTS_TEMPLATE.replace("[TESTDATA_TEST_NAME]", name)
130             test_results = test_results.replace("[TESTDATA_TEST_RESULTS]", test["results"])
131             test_results = test_results.replace("[TESTDATA_TEST_TIMES]", test["times"])
132             return test_results
133
134         test_results = JSON_RESULTS_DIRECTORY_TEMPLATE.replace("[TESTDATA_DIRECTORY]", name)
135         testdata = []
136         for (child_name, child_test) in sorted(test.iteritems()):
137             testdata.append(self._parse_tests_dict(child_name, child_test))
138         test_results = test_results.replace("[TESTDATA_DATA]", ",".join(testdata))
139         return test_results
140
141     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
142         aggregated_results = self._make_test_json(aggregated_data)
143         incremental_results = self._make_test_json(incremental_data)
144         merged_results = JsonResults.merge(self._builder,
145             aggregated_results, incremental_results, max_builds,
146             sort_keys=True)
147
148         if expected_data:
149             expected_results = self._make_test_json(expected_data)
150             self.assertEquals(merged_results, expected_results)
151         else:
152             self.assertFalse(merged_results)
153
154     def _test_get_test_list(self, input_data, expected_data):
155         input_results = self._make_test_json(input_data)
156
157         json_tests = []
158         for test in expected_data:
159             json_tests.append("\"" + test + "\":{}")
160
161         expected_results = JSON_RESULTS_PREFIX + \
162             JSON_RESULTS_TEST_LIST_TEMPLATE.replace(
163                 "[TESTDATA_TESTS]", ",".join(json_tests)) + \
164             JSON_RESULTS_SUFFIX
165
166         actual_results = JsonResults.get_test_list(self._builder, input_results)
167         self.assertEquals(actual_results, expected_results)
168
169     def test_merge_null_incremental_results(self):
170         # Empty incremental results json.
171         # Nothing to merge.
172         self._test_merge(
173             # Aggregated results
174             {"builds": ["2", "1"],
175              "tests": {"001.html": {
176                            "results": "[200,\"F\"]",
177                            "times": "[200,0]"}}},
178             # Incremental results
179             None,
180             # Expect no merge happens.
181             None)
182
183     def test_merge_empty_incremental_results(self):
184         # No actual incremental test results (only prefix and suffix) to merge.
185         # Nothing to merge.
186         self._test_merge(
187             # Aggregated results
188             {"builds": ["2", "1"],
189              "tests": {"001.html": {
190                            "results": "[200,\"F\"]",
191                            "times": "[200,0]"}}},
192             # Incremental results
193             {"builds": [],
194              "tests": {}},
195             # Expected no merge happens.
196             None)
197
198     def test_merge_empty_aggregated_results(self):
199         # No existing aggregated results.
200         # Merged results == new incremental results.
201         self._test_merge(
202             # Aggregated results
203             None,
204             # Incremental results
205
206             {"builds": ["2", "1"],
207              "tests": {"001.html": {
208                            "results": "[200,\"F\"]",
209                            "times": "[200,0]"}}},
210             # Expected result
211             {"builds": ["2", "1"],
212              "tests": {"001.html": {
213                            "results": "[200,\"F\"]",
214                            "times": "[200,0]"}}})
215
216     def test_merge_incremental_single_test_single_run_same_result(self):
217         # Incremental results has the latest build and same test results for
218         # that run.
219         # Insert the incremental results at the first place and sum number
220         # of runs for "F" (200 + 1) to get merged results.
221         self._test_merge(
222             # Aggregated results
223             {"builds": ["2", "1"],
224              "tests": {"001.html": {
225                            "results": "[200,\"F\"]",
226                            "times": "[200,0]"}}},
227             # Incremental results
228             {"builds": ["3"],
229              "tests": {"001.html": {
230                            "results": "[1,\"F\"]",
231                            "times": "[1,0]"}}},
232             # Expected results
233             {"builds": ["3", "2", "1"],
234              "tests": {"001.html": {
235                            "results": "[201,\"F\"]",
236                            "times": "[201,0]"}}})
237
238     def test_merge_single_test_single_run_different_result(self):
239         # Incremental results has the latest build but different test results
240         # for that run.
241         # Insert the incremental results at the first place.
242         self._test_merge(
243             # Aggregated results
244             {"builds": ["2", "1"],
245              "tests": {"001.html": {
246                            "results": "[200,\"F\"]",
247                            "times": "[200,0]"}}},
248             # Incremental results
249             {"builds": ["3"],
250              "tests": {"001.html": {
251                            "results": "[1, \"I\"]",
252                            "times": "[1,1]"}}},
253             # Expected results
254             {"builds": ["3", "2", "1"],
255              "tests": {"001.html": {
256                            "results": "[1,\"I\"],[200,\"F\"]",
257                            "times": "[1,1],[200,0]"}}})
258
259     def test_merge_single_test_single_run_result_changed(self):
260         # Incremental results has the latest build but results which differ from
261         # the latest result (but are the same as an older result).
262         self._test_merge(
263             # Aggregated results
264             {"builds": ["2", "1"],
265              "tests": {"001.html": {
266                            "results": "[200,\"F\"],[10,\"I\"]",
267                            "times": "[200,0],[10,1]"}}},
268             # Incremental results
269             {"builds": ["3"],
270              "tests": {"001.html": {
271                            "results": "[1,\"I\"]",
272                            "times": "[1,1]"}}},
273             # Expected results
274             {"builds": ["3", "2", "1"],
275              "tests": {"001.html": {
276                            "results": "[1,\"I\"],[200,\"F\"],[10,\"I\"]",
277                            "times": "[1,1],[200,0],[10,1]"}}})
278
279     def test_merge_multiple_tests_single_run(self):
280         # All tests have incremental updates.
281         self._test_merge(
282             # Aggregated results
283             {"builds": ["2", "1"],
284              "tests": {"001.html": {
285                            "results": "[200,\"F\"]",
286                            "times": "[200,0]"},
287                        "002.html": {
288                            "results": "[100,\"I\"]",
289                            "times": "[100,1]"}}},
290             # Incremental results
291             {"builds": ["3"],
292              "tests": {"001.html": {
293                            "results": "[1,\"F\"]",
294                            "times": "[1,0]"},
295                        "002.html": {
296                            "results": "[1,\"I\"]",
297                            "times": "[1,1]"}}},
298             # Expected results
299             {"builds": ["3", "2", "1"],
300              "tests": {"001.html": {
301                            "results": "[201,\"F\"]",
302                            "times": "[201,0]"},
303                        "002.html": {
304                            "results": "[101,\"I\"]",
305                            "times": "[101,1]"}}})
306
307     def test_merge_multiple_tests_single_run_one_no_result(self):
308         self._test_merge(
309             # Aggregated results
310             {"builds": ["2", "1"],
311              "tests": {"001.html": {
312                            "results": "[200,\"F\"]",
313                            "times": "[200,0]"},
314                        "002.html": {
315                            "results": "[100,\"I\"]",
316                            "times": "[100,1]"}}},
317             # Incremental results
318             {"builds": ["3"],
319              "tests": {"002.html": {
320                            "results": "[1,\"I\"]",
321                            "times": "[1,1]"}}},
322             # Expected results
323             {"builds": ["3", "2", "1"],
324              "tests": {"001.html": {
325                            "results": "[1,\"N\"],[200,\"F\"]",
326                            "times": "[201,0]"},
327                        "002.html": {
328                            "results": "[101,\"I\"]",
329                            "times": "[101,1]"}}})
330
331     def test_merge_single_test_multiple_runs(self):
332         self._test_merge(
333             # Aggregated results
334             {"builds": ["2", "1"],
335              "tests": {"001.html": {
336                            "results": "[200,\"F\"]",
337                            "times": "[200,0]"}}},
338             # Incremental results
339             {"builds": ["4", "3"],
340              "tests": {"001.html": {
341                            "results": "[2, \"I\"]",
342                            "times": "[2,2]"}}},
343             # Expected results
344             {"builds": ["4", "3", "2", "1"],
345              "tests": {"001.html": {
346                            "results": "[2,\"I\"],[200,\"F\"]",
347                            "times": "[2,2],[200,0]"}}})
348
349     def test_merge_multiple_tests_multiple_runs(self):
350         self._test_merge(
351             # Aggregated results
352             {"builds": ["2", "1"],
353              "tests": {"001.html": {
354                            "results": "[200,\"F\"]",
355                            "times": "[200,0]"},
356                        "002.html": {
357                            "results": "[10,\"Z\"]",
358                            "times": "[10,0]"}}},
359             # Incremental results
360             {"builds": ["4", "3"],
361              "tests": {"001.html": {
362                            "results": "[2, \"I\"]",
363                            "times": "[2,2]"},
364                        "002.html": {
365                            "results": "[1,\"C\"]",
366                            "times": "[1,1]"}}},
367             # Expected results
368             {"builds": ["4", "3", "2", "1"],
369              "tests": {"001.html": {
370                            "results": "[2,\"I\"],[200,\"F\"]",
371                            "times": "[2,2],[200,0]"},
372                        "002.html": {
373                            "results": "[1,\"C\"],[10,\"Z\"]",
374                            "times": "[1,1],[10,0]"}}})
375
376     def test_merge_incremental_result_older_build(self):
377         # Test the build in incremental results is older than the most recent
378         # build in aggregated results.
379         # The incremental results should be dropped and no merge happens.
380         self._test_merge(
381             # Aggregated results
382             {"builds": ["3", "1"],
383              "tests": {"001.html": {
384                            "results": "[200,\"F\"]",
385                            "times": "[200,0]"}}},
386             # Incremental results
387             {"builds": ["2"],
388              "tests": {"001.html": {
389                            "results": "[1, \"F\"]",
390                            "times": "[1,0]"}}},
391             # Expected no merge happens.
392             None)
393
394     def test_merge_incremental_result_same_build(self):
395         # Test the build in incremental results is same as the build in
396         # aggregated results.
397         # The incremental results should be dropped and no merge happens.
398         self._test_merge(
399             # Aggregated results
400             {"builds": ["2", "1"],
401              "tests": {"001.html": {
402                            "results": "[200,\"F\"]",
403                            "times": "[200,0]"}}},
404             # Incremental results
405             {"builds": ["3", "2"],
406              "tests": {"001.html": {
407                            "results": "[2, \"F\"]",
408                            "times": "[2,0]"}}},
409             # Expected no merge happens.
410             None)
411
412     def test_merge_remove_test_with_no_data(self):
413         # Remove test where there is no data in all runs.
414         self._test_merge(
415             # Aggregated results
416             {"builds": ["2", "1"],
417              "tests": {"001.html": {
418                            "results": "[200,\"N\"]",
419                            "times": "[200,0]"},
420                        "002.html": {
421                            "results": "[10,\"F\"]",
422                            "times": "[10,0]"}}},
423             # Incremental results
424             {"builds": ["3"],
425              "tests": {"001.html": {
426                            "results": "[1,\"N\"]",
427                            "times": "[1,0]"},
428                        "002.html": {
429                            "results": "[1,\"P\"]",
430                            "times": "[1,0]"}}},
431             # Expected results
432             {"builds": ["3", "2", "1"],
433              "tests": {"002.html": {
434                            "results": "[1,\"P\"],[10,\"F\"]",
435                            "times": "[11,0]"}}})
436
437     def test_merge_remove_test_with_all_pass(self):
438         # Remove test where all run pass and max running time < 1 seconds
439         self._test_merge(
440             # Aggregated results
441             {"builds": ["2", "1"],
442              "tests": {"001.html": {
443                            "results": "[200,\"P\"]",
444                            "times": "[200,0]"},
445                        "002.html": {
446                            "results": "[10,\"F\"]",
447                            "times": "[10,0]"}}},
448             # Incremental results
449             {"builds": ["3"],
450              "tests": {"001.html": {
451                            "results": "[1,\"P\"]",
452                            "times": "[1,0]"},
453                        "002.html": {
454                            "results": "[1,\"P\"]",
455                            "times": "[1,0]"}}},
456             # Expected results
457             {"builds": ["3", "2", "1"],
458              "tests": {"002.html": {
459                            "results": "[1,\"P\"],[10,\"F\"]",
460                            "times": "[11,0]"}}})
461
462     def test_merge_keep_test_with_all_pass_but_slow_time(self):
463         # Do not remove test where all run pass but max running time >= 1 seconds
464         self._test_merge(
465             # Aggregated results
466             {"builds": ["2", "1"],
467              "tests": {"001.html": {
468                            "results": "[200,\"P\"]",
469                            "times": "[200,0]"},
470                        "002.html": {
471                            "results": "[10,\"F\"]",
472                            "times": "[10,0]"}}},
473             # Incremental results
474             {"builds": ["3"],
475              "tests": {"001.html": {
476                            "results": "[1,\"P\"]",
477                            "times": "[1,1]"},
478                        "002.html": {
479                            "results": "[1,\"P\"]",
480                            "times": "[1,0]"}}},
481             # Expected results
482             {"builds": ["3", "2", "1"],
483              "tests": {"001.html": {
484                            "results": "[201,\"P\"]",
485                            "times": "[1,1],[200,0]"},
486                        "002.html": {
487                            "results": "[1,\"P\"],[10,\"F\"]",
488                            "times": "[11,0]"}}})
489
490     def test_merge_prune_extra_results(self):
491         # Remove items from test results and times that exceed the max number
492         # of builds to track.
493         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS)
494         self._test_merge(
495             # Aggregated results
496             {"builds": ["2", "1"],
497              "tests": {"001.html": {
498                            "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
499                            "times": "[" + max_builds + ",0],[1,1]"}}},
500             # Incremental results
501             {"builds": ["3"],
502              "tests": {"001.html": {
503                            "results": "[1,\"T\"]",
504                            "times": "[1,1]"}}},
505             # Expected results
506             {"builds": ["3", "2", "1"],
507              "tests": {"001.html": {
508                            "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
509                            "times": "[1,1],[" + max_builds + ",0]"}}})
510
511     def test_merge_prune_extra_results_small(self):
512         # Remove items from test results and times that exceed the max number
513         # of builds to track, using smaller threshold.
514         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
515         self._test_merge(
516             # Aggregated results
517             {"builds": ["2", "1"],
518              "tests": {"001.html": {
519                            "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
520                            "times": "[" + max_builds + ",0],[1,1]"}}},
521             # Incremental results
522             {"builds": ["3"],
523              "tests": {"001.html": {
524                            "results": "[1,\"T\"]",
525                            "times": "[1,1]"}}},
526             # Expected results
527             {"builds": ["3", "2", "1"],
528              "tests": {"001.html": {
529                            "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
530                            "times": "[1,1],[" + max_builds + ",0]"}}},
531             int(max_builds))
532
533     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
534         # Test that merging in a new result of the same type as the last result
535         # causes old results to fall off.
536         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
537         self._test_merge(
538             # Aggregated results
539             {"builds": ["2", "1"],
540              "tests": {"001.html": {
541                            "results": "[" + max_builds + ",\"F\"],[1,\"N\"]",
542                            "times": "[" + max_builds + ",0],[1,1]"}}},
543             # Incremental results
544             {"builds": ["3"],
545              "tests": {"001.html": {
546                            "results": "[1,\"F\"]",
547                            "times": "[1,0]"}}},
548             # Expected results
549             {"builds": ["3", "2", "1"],
550              "tests": {"001.html": {
551                            "results": "[" + max_builds + ",\"F\"]",
552                            "times": "[" + max_builds + ",0]"}}},
553             int(max_builds))
554
555     def test_merge_build_directory_hierarchy(self):
556         self._test_merge(
557             # Aggregated results
558             {"builds": ["2", "1"],
559              "tests": {"foo/001.html": {
560                            "results": "[50,\"F\"]",
561                            "times": "[50,0]"},
562                        "foo/002.html": {
563                            "results": "[100,\"I\"]",
564                            "times": "[100,0]"}}},
565             # Incremental results
566             {"builds": ["3"],
567              "tests": {"foo": {
568                            "001.html": {
569                                "results": "[1,\"F\"]",
570                                "times": "[1,0]"},
571                            "002.html": {
572                                "results": "[1,\"I\"]",
573                                "times": "[1,0]"}}},
574              "version": 4},
575             # Expected results
576             {"builds": ["3", "2", "1"],
577              "tests": {"foo/001.html": {
578                            "results": "[51,\"F\"]",
579                            "times": "[51,0]"},
580                        "foo/002.html": {
581                            "results": "[101,\"I\"]",
582                            "times": "[101,0]"}},
583              "version": 3})
584
585     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
586
587     def test_get_test_name_list(self):
588         # Get test name list only. Don't include non-test-list data and
589         # of test result details.
590         self._test_get_test_list(
591             # Input results
592             {"builds": ["3", "2", "1"],
593              "tests": {"001.html": {
594                            "results": "[200,\"P\"]",
595                            "times": "[200,0]"},
596                        "002.html": {
597                            "results": "[10,\"F\"]",
598                            "times": "[10,0]"}}},
599             # Expected results
600             ["001.html", "002.html"])
601
602 if __name__ == '__main__':
603     unittest.main()