Allow json NRWT downloads to be pure json and not jsonp
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / json_results_generator_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """Unit tests for json_results_generator.py."""
30
31 import unittest
32 import optparse
33 import random
34
35 from webkitpy.common.system import filesystem_mock
36 from webkitpy.layout_tests.layout_package import json_results_generator
37 from webkitpy.layout_tests.models import test_expectations
38 from webkitpy.layout_tests.port import test
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.thirdparty import simplejson
41
42 class JSONGeneratorTest(unittest.TestCase):
43     def setUp(self):
44         self.builder_name = 'DUMMY_BUILDER_NAME'
45         self.build_name = 'DUMMY_BUILD_NAME'
46         self.build_number = 'DUMMY_BUILDER_NUMBER'
47
48         # For archived results.
49         self._json = None
50         self._num_runs = 0
51         self._tests_set = set([])
52         self._test_timings = {}
53         self._failed_count_map = {}
54
55         self._PASS_count = 0
56         self._DISABLED_count = 0
57         self._FLAKY_count = 0
58         self._FAILS_count = 0
59         self._fixable_count = 0
60
61     def test_strip_json_wrapper(self):
62         json = "['contents']"
63         self.assertEqual(json_results_generator.strip_json_wrapper(json_results_generator._JSON_PREFIX + json + json_results_generator._JSON_SUFFIX), json)
64         self.assertEqual(json_results_generator.strip_json_wrapper(json), json)
65
66     def _test_json_generation(self, passed_tests_list, failed_tests_list):
67         tests_set = set(passed_tests_list) | set(failed_tests_list)
68
69         DISABLED_tests = set([t for t in tests_set
70                              if t.startswith('DISABLED_')])
71         FLAKY_tests = set([t for t in tests_set
72                            if t.startswith('FLAKY_')])
73         FAILS_tests = set([t for t in tests_set
74                            if t.startswith('FAILS_')])
75         PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
76
77         failed_tests = set(failed_tests_list) - DISABLED_tests
78         failed_count_map = dict([(t, 1) for t in failed_tests])
79
80         test_timings = {}
81         i = 0
82         for test in tests_set:
83             test_timings[test] = float(self._num_runs * 100 + i)
84             i += 1
85
86         test_results_map = dict()
87         for test in tests_set:
88             test_results_map[test] = json_results_generator.TestResult(test,
89                 failed=(test in failed_tests),
90                 elapsed_time=test_timings[test])
91
92         port = Mock()
93         port._filesystem = filesystem_mock.MockFileSystem()
94         generator = json_results_generator.JSONResultsGeneratorBase(port,
95             self.builder_name, self.build_name, self.build_number,
96             '',
97             None,   # don't fetch past json results archive
98             test_results_map)
99
100         failed_count_map = dict([(t, 1) for t in failed_tests])
101
102         # Test incremental json results
103         incremental_json = generator.get_json()
104         self._verify_json_results(
105             tests_set,
106             test_timings,
107             failed_count_map,
108             len(PASS_tests),
109             len(DISABLED_tests),
110             len(FLAKY_tests),
111             len(DISABLED_tests | failed_tests),
112             incremental_json,
113             1)
114
115         # We don't verify the results here, but at least we make sure the code runs without errors.
116         generator.generate_json_output()
117         generator.generate_times_ms_file()
118
119     def _verify_json_results(self, tests_set, test_timings, failed_count_map,
120                              PASS_count, DISABLED_count, FLAKY_count,
121                              fixable_count,
122                              json, num_runs):
123         # Aliasing to a short name for better access to its constants.
124         JRG = json_results_generator.JSONResultsGeneratorBase
125
126         self.assertTrue(JRG.VERSION_KEY in json)
127         self.assertTrue(self.builder_name in json)
128
129         buildinfo = json[self.builder_name]
130         self.assertTrue(JRG.FIXABLE in buildinfo)
131         self.assertTrue(JRG.TESTS in buildinfo)
132         self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs)
133         self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number)
134
135         if tests_set or DISABLED_count:
136             fixable = {}
137             for fixable_items in buildinfo[JRG.FIXABLE]:
138                 for (type, count) in fixable_items.iteritems():
139                     if type in fixable:
140                         fixable[type] = fixable[type] + count
141                     else:
142                         fixable[type] = count
143
144             if PASS_count:
145                 self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count)
146             else:
147                 self.assertTrue(JRG.PASS_RESULT not in fixable or
148                                 fixable[JRG.PASS_RESULT] == 0)
149             if DISABLED_count:
150                 self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count)
151             else:
152                 self.assertTrue(JRG.SKIP_RESULT not in fixable or
153                                 fixable[JRG.SKIP_RESULT] == 0)
154             if FLAKY_count:
155                 self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count)
156             else:
157                 self.assertTrue(JRG.FLAKY_RESULT not in fixable or
158                                 fixable[JRG.FLAKY_RESULT] == 0)
159
160         if failed_count_map:
161             tests = buildinfo[JRG.TESTS]
162             for test_name in failed_count_map.iterkeys():
163                 test = self._find_test_in_trie(test_name, tests)
164
165                 failed = 0
166                 for result in test[JRG.RESULTS]:
167                     if result[1] == JRG.FAIL_RESULT:
168                         failed += result[0]
169                 self.assertEqual(failed_count_map[test_name], failed)
170
171                 timing_count = 0
172                 for timings in test[JRG.TIMES]:
173                     if timings[1] == test_timings[test_name]:
174                         timing_count = timings[0]
175                 self.assertEqual(1, timing_count)
176
177         if fixable_count:
178             self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count)
179
180     def _find_test_in_trie(self, path, trie):
181         nodes = path.split("/")
182         sub_trie = trie
183         for node in nodes:
184             self.assertTrue(node in sub_trie)
185             sub_trie = sub_trie[node]
186         return sub_trie
187
188     def test_json_generation(self):
189         self._test_json_generation([], [])
190         self._test_json_generation(['A1', 'B1'], [])
191         self._test_json_generation([], ['FAILS_A2', 'FAILS_B2'])
192         self._test_json_generation(['DISABLED_A3', 'DISABLED_B3'], [])
193         self._test_json_generation(['A4'], ['B4', 'FAILS_C4'])
194         self._test_json_generation(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5'])
195         self._test_json_generation(
196             ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'],
197             ['FAILS_D6'])
198
199         # Generate JSON with the same test sets. (Both incremental results and
200         # archived results must be updated appropriately.)
201         self._test_json_generation(
202             ['A', 'FLAKY_B', 'DISABLED_C'],
203             ['FAILS_D', 'FLAKY_E'])
204         self._test_json_generation(
205             ['A', 'DISABLED_C', 'FLAKY_E'],
206             ['FLAKY_B', 'FAILS_D'])
207         self._test_json_generation(
208             ['FLAKY_B', 'DISABLED_C', 'FAILS_D'],
209             ['A', 'FLAKY_E'])
210
211     def test_hierarchical_json_generation(self):
212         # FIXME: Re-work tests to be more comprehensible and comprehensive.
213         self._test_json_generation(['foo/A'], ['foo/B', 'bar/C'])
214
215     def test_test_timings_trie(self):
216         test_port = test.TestPort()
217         individual_test_timings = []
218         individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2))
219         individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
220         trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
221
222         expected_trie = {
223           'bar.html': 0,
224           'foo': {
225               'bar': {
226                   'baz.html': 1200,
227               }
228           }
229         }
230
231         self.assertEqual(simplejson.dumps(trie), simplejson.dumps(expected_trie))
232
233
234
235 if __name__ == '__main__':
236     unittest.main()