testlistjson on the test results server doesn't understand hierarchical results format
[WebKit-https.git] / Tools / TestResultServer / model / jsonresults.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 from datetime import datetime
30 from django.utils import simplejson
31 import logging
32
33 from model.testfile import TestFile
34
35 JSON_RESULTS_FILE = "results.json"
36 JSON_RESULTS_FILE_SMALL = "results-small.json"
37 JSON_RESULTS_PREFIX = "ADD_RESULTS("
38 JSON_RESULTS_SUFFIX = ");"
39 JSON_RESULTS_VERSION_KEY = "version"
40 JSON_RESULTS_BUILD_NUMBERS = "buildNumbers"
41 JSON_RESULTS_TESTS = "tests"
42 JSON_RESULTS_RESULTS = "results"
43 JSON_RESULTS_TIMES = "times"
44 JSON_RESULTS_PASS = "P"
45 JSON_RESULTS_NO_DATA = "N"
46 JSON_RESULTS_MIN_TIME = 1
47 JSON_RESULTS_HIERARCHICAL_VERSION = 4
48 JSON_RESULTS_MAX_BUILDS = 750
49 JSON_RESULTS_MAX_BUILDS_SMALL = 200
50
51
52 def _add_path_to_trie(path, value, trie):
53     if not "/" in path:
54         trie[path] = value
55         return
56
57     directory, slash, rest = path.partition("/")
58     if not directory in trie:
59         trie[directory] = {}
60     _add_path_to_trie(rest, value, trie[directory])
61
62
63 def _trie_json_tests(tests):
64     """Breaks a test name into chunks by directory and puts the test time as a value in the lowest part, e.g.
65     foo/bar/baz.html: VALUE1
66     foo/bar/baz1.html: VALUE2
67
68     becomes
69     foo: {
70         bar: {
71             baz.html: VALUE1,
72             baz1.html: VALUE2
73         }
74     }
75     """
76     trie = {}
77     for test, value in tests.iteritems():
78         _add_path_to_trie(test, value, trie)
79     return trie
80
81
82 class JsonResults(object):
83     @classmethod
84     def _strip_prefix_suffix(cls, data):
85         # FIXME: Stop stripping jsonp callback once we upload pure json everywhere.
86         if data.startswith(JSON_RESULTS_PREFIX) and data.endswith(JSON_RESULTS_SUFFIX):
87             return data[len(JSON_RESULTS_PREFIX):len(data) - len(JSON_RESULTS_SUFFIX)]
88         return data
89
90     @classmethod
91     def _generate_file_data(cls, json, sort_keys=False):
92         return simplejson.dumps(json, separators=(',', ':'), sort_keys=sort_keys)
93
94     @classmethod
95     def _load_json(cls, file_data):
96         json_results_str = cls._strip_prefix_suffix(file_data)
97         if not json_results_str:
98             logging.warning("No json results data.")
99             return None
100
101         try:
102             return simplejson.loads(json_results_str)
103         except Exception, err:
104             logging.debug(json_results_str)
105             logging.error("Failed to load json results: %s", str(err))
106             return None
107
108     @classmethod
109     def _merge_json(cls, aggregated_json, incremental_json, num_runs):
110         cls._merge_non_test_data(aggregated_json, incremental_json, num_runs)
111         incremental_tests = incremental_json[JSON_RESULTS_TESTS]
112         if incremental_tests:
113             aggregated_tests = aggregated_json[JSON_RESULTS_TESTS]
114             cls._merge_tests(aggregated_tests, incremental_tests, num_runs)
115
116     @classmethod
117     def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs):
118         incremental_builds = incremental_json[JSON_RESULTS_BUILD_NUMBERS]
119         aggregated_builds = aggregated_json[JSON_RESULTS_BUILD_NUMBERS]
120         aggregated_build_number = int(aggregated_builds[0])
121
122         for index in reversed(range(len(incremental_builds))):
123             build_number = int(incremental_builds[index])
124             logging.debug("Merging build %s, incremental json index: %d.", build_number, index)
125
126             # Merge this build into aggreagated results.
127             cls._merge_one_build(aggregated_json, incremental_json, index, num_runs)
128
129     @classmethod
130     def _merge_one_build(cls, aggregated_json, incremental_json, incremental_index, num_runs):
131         for key in incremental_json.keys():
132             # Merge json results except "tests" properties (results, times etc).
133             # "tests" properties will be handled separately.
134             if key == JSON_RESULTS_TESTS:
135                 continue
136
137             if key in aggregated_json:
138                 aggregated_json[key].insert(0, incremental_json[key][incremental_index])
139                 aggregated_json[key] = aggregated_json[key][:num_runs]
140             else:
141                 aggregated_json[key] = incremental_json[key]
142
143     @classmethod
144     def _merge_tests(cls, aggregated_json, incremental_json, num_runs):
145         all_tests = set(aggregated_json.iterkeys())
146         if incremental_json:
147             all_tests |= set(incremental_json.iterkeys())
148
149         for test_name in all_tests:
150             if test_name not in aggregated_json:
151                 aggregated_json[test_name] = incremental_json[test_name]
152                 continue
153
154             incremental_sub_result = incremental_json[test_name] if incremental_json and test_name in incremental_json else None
155             if JSON_RESULTS_RESULTS not in aggregated_json[test_name]:
156                 cls._merge_tests(aggregated_json[test_name], incremental_sub_result, num_runs)
157                 continue
158
159             if incremental_sub_result:
160                 results = incremental_sub_result[JSON_RESULTS_RESULTS]
161                 times = incremental_sub_result[JSON_RESULTS_TIMES]
162             else:
163                 results = [[1, JSON_RESULTS_NO_DATA]]
164                 times = [[1, 0]]
165
166             aggregated_test = aggregated_json[test_name]
167             cls._insert_item_run_length_encoded(results, aggregated_test[JSON_RESULTS_RESULTS], num_runs)
168             cls._insert_item_run_length_encoded(times, aggregated_test[JSON_RESULTS_TIMES], num_runs)
169             cls._normalize_results_json(test_name, aggregated_json, num_runs)
170
171     @classmethod
172     def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs):
173         for item in incremental_item:
174             if len(aggregated_item) and item[1] == aggregated_item[0][1]:
175                 aggregated_item[0][0] = min(aggregated_item[0][0] + item[0], num_runs)
176             else:
177                 aggregated_item.insert(0, item)
178
179     @classmethod
180     def _normalize_results_json(cls, test_name, aggregated_json, num_runs):
181         aggregated_test = aggregated_json[test_name]
182         aggregated_test[JSON_RESULTS_RESULTS] = cls._remove_items_over_max_number_of_builds(aggregated_test[JSON_RESULTS_RESULTS], num_runs)
183         aggregated_test[JSON_RESULTS_TIMES] = cls._remove_items_over_max_number_of_builds(aggregated_test[JSON_RESULTS_TIMES], num_runs)
184
185         is_all_pass = cls._is_results_all_of_type(aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_PASS)
186         is_all_no_data = cls._is_results_all_of_type(aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_NO_DATA)
187
188         max_time = max([time[1] for time in aggregated_test[JSON_RESULTS_TIMES]])
189         if (is_all_no_data or (is_all_pass and max_time < JSON_RESULTS_MIN_TIME)):
190             del aggregated_json[test_name]
191
192     @classmethod
193     def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs):
194         num_builds = 0
195         index = 0
196         for result in encoded_list:
197             num_builds = num_builds + result[0]
198             index = index + 1
199             if num_builds >= num_runs:
200                 return encoded_list[:index]
201
202         return encoded_list
203
204     @classmethod
205     def _is_results_all_of_type(cls, results, type):
206         return len(results) == 1 and results[0][1] == type
207
208     @classmethod
209     def _remove_gtest_modifiers(cls, builder, json):
210         tests = json[builder][JSON_RESULTS_TESTS]
211         new_tests = {}
212         # FIXME: This is wrong. If the test exists in the incremental results as both values, then one will overwrite the other.
213         # We should instead pick the one that doesn't have NO_DATA as its value.
214         # Alternately we could fix this by having the JSON generation code on the buildbot only include the test
215         # that was actually run.
216         for name, test in tests.iteritems():
217             new_name = name.replace('.FLAKY_', '.', 1)
218             new_name = new_name.replace('.FAILS_', '.', 1)
219             new_name = new_name.replace('.MAYBE_', '.', 1)
220             new_name = new_name.replace('.DISABLED_', '.', 1)
221             new_tests[new_name] = test
222         json[builder][JSON_RESULTS_TESTS] = new_tests
223
224     @classmethod
225     def _check_json(cls, builder, json):
226         version = json[JSON_RESULTS_VERSION_KEY]
227         if version > JSON_RESULTS_HIERARCHICAL_VERSION:
228             logging.error("Results JSON version '%s' is not supported.",
229                 version)
230             return False
231
232         if not builder in json:
233             logging.error("Builder '%s' is not in json results.", builder)
234             return False
235
236         results_for_builder = json[builder]
237         if not JSON_RESULTS_BUILD_NUMBERS in results_for_builder:
238             logging.error("Missing build number in json results.")
239             return False
240
241         # FIXME: Once all the bots have cycled, we can remove this code since all the results will be heirarchical.
242         if version < JSON_RESULTS_HIERARCHICAL_VERSION:
243             json[builder][JSON_RESULTS_TESTS] = _trie_json_tests(results_for_builder[JSON_RESULTS_TESTS])
244             json[JSON_RESULTS_VERSION_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION
245
246         return True
247
248     @classmethod
249     def merge(cls, builder, aggregated, incremental, num_runs, sort_keys=False):
250         if not incremental:
251             logging.warning("Nothing to merge.")
252             return None
253
254         logging.info("Loading incremental json...")
255         incremental_json = cls._load_json(incremental)
256         if not incremental_json:
257             return None
258
259         logging.info("Checking incremental json...")
260         if not cls._check_json(builder, incremental_json):
261             return None
262
263         # FIXME: We should probably avoid doing this for layout tests.
264         cls._remove_gtest_modifiers(builder, incremental_json)
265
266         logging.info("Loading existing aggregated json...")
267         aggregated_json = cls._load_json(aggregated)
268         if not aggregated_json:
269             return incremental
270
271         logging.info("Checking existing aggregated json...")
272         if not cls._check_json(builder, aggregated_json):
273             return incremental
274
275         logging.info("Merging json results...")
276         try:
277             cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs)
278         except Exception, err:
279             logging.error("Failed to merge json results: %s", str(err))
280             return None
281
282         aggregated_json[JSON_RESULTS_VERSION_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION
283
284         return cls._generate_file_data(aggregated_json, sort_keys)
285
286     @classmethod
287     def update(cls, master, builder, test_type, incremental):
288         small_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE_SMALL, JSON_RESULTS_MAX_BUILDS_SMALL)
289         large_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE, JSON_RESULTS_MAX_BUILDS)
290
291         return small_file_updated and large_file_updated
292
293     @classmethod
294     def update_file(cls, master, builder, test_type, incremental, filename, num_runs):
295         files = TestFile.get_files(master, builder, test_type, filename)
296         if files:
297             file = files[0]
298             new_results = cls.merge(builder, file.data, incremental, num_runs)
299         else:
300             # Use the incremental data if there is no aggregated file to merge.
301             file = TestFile()            
302             file.master = master
303             file.builder = builder
304             file.test_type = test_type
305             file.name = filename
306             new_results = incremental
307             logging.info("No existing json results, incremental json is saved.")
308
309         if not new_results or not file.save(new_results):
310             logging.info("Update failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, filename))
311             return False
312
313         return True
314
315     @classmethod
316     def _delete_results_and_times(cls, tests):
317         for key in tests.keys():
318             if key in (JSON_RESULTS_RESULTS, JSON_RESULTS_TIMES):
319                 del tests[key]
320             else:
321                 cls._delete_results_and_times(tests[key])
322
323     @classmethod
324     def get_test_list(cls, builder, json_file_data):
325         logging.debug("Loading test results json...")
326         json = cls._load_json(json_file_data)
327         if not json:
328             return None
329
330         logging.debug("Checking test results json...")
331         if not cls._check_json(builder, json):
332             return None
333
334         test_list_json = {}
335         tests = json[builder][JSON_RESULTS_TESTS]
336         cls._delete_results_and_times(tests)
337         test_list_json[builder] = {"tests": tests}
338         return cls._generate_file_data(test_list_json)