Rename WebKitTools to Tools
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / json_results_generator_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """Unit tests for json_results_generator.py."""
30
31 import unittest
32 import optparse
33 import random
34 import shutil
35 import tempfile
36
37 from webkitpy.layout_tests.layout_package import json_results_generator
38 from webkitpy.layout_tests.layout_package import test_expectations
39
40
41 class JSONGeneratorTest(unittest.TestCase):
42     def setUp(self):
43         self.builder_name = 'DUMMY_BUILDER_NAME'
44         self.build_name = 'DUMMY_BUILD_NAME'
45         self.build_number = 'DUMMY_BUILDER_NUMBER'
46
47         # For archived results.
48         self._json = None
49         self._num_runs = 0
50         self._tests_set = set([])
51         self._test_timings = {}
52         self._failed_count_map = {}
53
54         self._PASS_count = 0
55         self._DISABLED_count = 0
56         self._FLAKY_count = 0
57         self._FAILS_count = 0
58         self._fixable_count = 0
59
60     def _test_json_generation(self, passed_tests_list, failed_tests_list):
61         tests_set = set(passed_tests_list) | set(failed_tests_list)
62
63         DISABLED_tests = set([t for t in tests_set
64                              if t.startswith('DISABLED_')])
65         FLAKY_tests = set([t for t in tests_set
66                            if t.startswith('FLAKY_')])
67         FAILS_tests = set([t for t in tests_set
68                            if t.startswith('FAILS_')])
69         PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
70
71         failed_tests = set(failed_tests_list) - DISABLED_tests
72         failed_count_map = dict([(t, 1) for t in failed_tests])
73
74         test_timings = {}
75         i = 0
76         for test in tests_set:
77             test_timings[test] = float(self._num_runs * 100 + i)
78             i += 1
79
80         test_results_map = dict()
81         for test in tests_set:
82             test_results_map[test] = json_results_generator.TestResult(test,
83                 failed=(test in failed_tests),
84                 elapsed_time=test_timings[test])
85
86         generator = json_results_generator.JSONResultsGeneratorBase(
87             self.builder_name, self.build_name, self.build_number,
88             '',
89             None,   # don't fetch past json results archive
90             test_results_map)
91
92         failed_count_map = dict([(t, 1) for t in failed_tests])
93
94         # Test incremental json results
95         incremental_json = generator.get_json(incremental=True)
96         self._verify_json_results(
97             tests_set,
98             test_timings,
99             failed_count_map,
100             len(PASS_tests),
101             len(DISABLED_tests),
102             len(FLAKY_tests),
103             len(DISABLED_tests | failed_tests),
104             incremental_json,
105             1)
106
107         # Test aggregated json results
108         generator.set_archived_results(self._json)
109         json = generator.get_json(incremental=False)
110         self._json = json
111         self._num_runs += 1
112         self._tests_set |= tests_set
113         self._test_timings.update(test_timings)
114         self._PASS_count += len(PASS_tests)
115         self._DISABLED_count += len(DISABLED_tests)
116         self._FLAKY_count += len(FLAKY_tests)
117         self._fixable_count += len(DISABLED_tests | failed_tests)
118
119         get = self._failed_count_map.get
120         for test in failed_count_map.iterkeys():
121             self._failed_count_map[test] = get(test, 0) + 1
122
123         self._verify_json_results(
124             self._tests_set,
125             self._test_timings,
126             self._failed_count_map,
127             self._PASS_count,
128             self._DISABLED_count,
129             self._FLAKY_count,
130             self._fixable_count,
131             self._json,
132             self._num_runs)
133
134     def _verify_json_results(self, tests_set, test_timings, failed_count_map,
135                              PASS_count, DISABLED_count, FLAKY_count,
136                              fixable_count,
137                              json, num_runs):
138         # Aliasing to a short name for better access to its constants.
139         JRG = json_results_generator.JSONResultsGeneratorBase
140
141         self.assertTrue(JRG.VERSION_KEY in json)
142         self.assertTrue(self.builder_name in json)
143
144         buildinfo = json[self.builder_name]
145         self.assertTrue(JRG.FIXABLE in buildinfo)
146         self.assertTrue(JRG.TESTS in buildinfo)
147         self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs)
148         self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number)
149
150         if tests_set or DISABLED_count:
151             fixable = {}
152             for fixable_items in buildinfo[JRG.FIXABLE]:
153                 for (type, count) in fixable_items.iteritems():
154                     if type in fixable:
155                         fixable[type] = fixable[type] + count
156                     else:
157                         fixable[type] = count
158
159             if PASS_count:
160                 self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count)
161             else:
162                 self.assertTrue(JRG.PASS_RESULT not in fixable or
163                                 fixable[JRG.PASS_RESULT] == 0)
164             if DISABLED_count:
165                 self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count)
166             else:
167                 self.assertTrue(JRG.SKIP_RESULT not in fixable or
168                                 fixable[JRG.SKIP_RESULT] == 0)
169             if FLAKY_count:
170                 self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count)
171             else:
172                 self.assertTrue(JRG.FLAKY_RESULT not in fixable or
173                                 fixable[JRG.FLAKY_RESULT] == 0)
174
175         if failed_count_map:
176             tests = buildinfo[JRG.TESTS]
177             for test_name in failed_count_map.iterkeys():
178                 self.assertTrue(test_name in tests)
179                 test = tests[test_name]
180
181                 failed = 0
182                 for result in test[JRG.RESULTS]:
183                     if result[1] == JRG.FAIL_RESULT:
184                         failed += result[0]
185                 self.assertEqual(failed_count_map[test_name], failed)
186
187                 timing_count = 0
188                 for timings in test[JRG.TIMES]:
189                     if timings[1] == test_timings[test_name]:
190                         timing_count = timings[0]
191                 self.assertEqual(1, timing_count)
192
193         if fixable_count:
194             self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count)
195
196     def test_json_generation(self):
197         self._test_json_generation([], [])
198         self._test_json_generation(['A1', 'B1'], [])
199         self._test_json_generation([], ['FAILS_A2', 'FAILS_B2'])
200         self._test_json_generation(['DISABLED_A3', 'DISABLED_B3'], [])
201         self._test_json_generation(['A4'], ['B4', 'FAILS_C4'])
202         self._test_json_generation(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5'])
203         self._test_json_generation(
204             ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'],
205             ['FAILS_D6'])
206
207         # Generate JSON with the same test sets. (Both incremental results and
208         # archived results must be updated appropriately.)
209         self._test_json_generation(
210             ['A', 'FLAKY_B', 'DISABLED_C'],
211             ['FAILS_D', 'FLAKY_E'])
212         self._test_json_generation(
213             ['A', 'DISABLED_C', 'FLAKY_E'],
214             ['FLAKY_B', 'FAILS_D'])
215         self._test_json_generation(
216             ['FLAKY_B', 'DISABLED_C', 'FAILS_D'],
217             ['A', 'FLAKY_E'])
218
219 if __name__ == '__main__':
220     unittest.main()