testlistjson on the test results server doesn't understand hierarchical results format
[WebKit-https.git] / Tools / TestResultServer / model / jsonresults_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 try:
30     import jsonresults
31     from jsonresults import JsonResults
32 except ImportError:
33     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
34     raise
35
36 from django.utils import simplejson
37
38 import unittest
39
40
41 JSON_RESULTS_TEMPLATE = (
42     '{"Webkit":{'
43     '"allFixableCount":[[TESTDATA_COUNT]],'
44     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
45     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
46     '"deferredCounts":[[TESTDATA_COUNTS]],'
47     '"fixableCount":[[TESTDATA_COUNT]],'
48     '"fixableCounts":[[TESTDATA_COUNTS]],'
49     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
50     '"tests":{[TESTDATA_TESTS]},'
51     '"webkitRevision":[[TESTDATA_WEBKITREVISION]],'
52     '"wontfixCounts":[[TESTDATA_COUNTS]]'
53     '},'
54     '"version":[VERSION]'
55     '}')
56
57 JSON_RESULTS_COUNTS_TEMPLATE = (
58     '{'
59     '"C":[TESTDATA],'
60     '"F":[TESTDATA],'
61     '"I":[TESTDATA],'
62     '"O":[TESTDATA],'
63     '"P":[TESTDATA],'
64     '"T":[TESTDATA],'
65     '"X":[TESTDATA],'
66     '"Z":[TESTDATA]}')
67
68 JSON_RESULTS_DIRECTORY_TEMPLATE = '[[TESTDATA_DIRECTORY]]:{[TESTDATA_DATA]}'
69
70 JSON_RESULTS_TESTS_TEMPLATE = (
71     '[[TESTDATA_TEST_NAME]]:{'
72     '"results":[[TESTDATA_TEST_RESULTS]],'
73     '"times":[[TESTDATA_TEST_TIMES]]}')
74
75 JSON_RESULTS_TEST_LIST_TEMPLATE = (
76     '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}')
77
78
79 class JsonResultsTest(unittest.TestCase):
80     def setUp(self):
81         self._builder = "Webkit"
82
83     def test_strip_prefix_suffix(self):
84         json = "['contents']"
85         self.assertEqual(JsonResults._strip_prefix_suffix("ADD_RESULTS(" + json + ");"), json)
86         self.assertEqual(JsonResults._strip_prefix_suffix(json), json)
87
88     def _make_test_json(self, test_data):
89         if not test_data:
90             return ""
91
92         builds = test_data["builds"]
93         tests = test_data["tests"]
94         if not builds or not tests:
95             return ""
96
97         json = JSON_RESULTS_TEMPLATE
98
99         counts = []
100         build_numbers = []
101         webkit_revision = []
102         chrome_revision = []
103         times = []
104         for build in builds:
105             counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
106             build_numbers.append("1000%s" % build)
107             webkit_revision.append("2000%s" % build)
108             chrome_revision.append("3000%s" % build)
109             times.append("100000%s000" % build)
110
111         json = json.replace("[TESTDATA_COUNTS]", ",".join(counts))
112         json = json.replace("[TESTDATA_COUNT]", ",".join(builds))
113         json = json.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
114         json = json.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
115         json = json.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
116         json = json.replace("[TESTDATA_TIMES]", ",".join(times))
117
118         version = str(test_data["version"]) if "version" in test_data else "4"
119         json = json.replace("[VERSION]", version)
120         json = json.replace("{[TESTDATA_TESTS]}", simplejson.dumps(tests, separators=(',', ':'), sort_keys=True))
121         return json
122
123     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
124         aggregated_results = self._make_test_json(aggregated_data)
125         incremental_results = self._make_test_json(incremental_data)
126         merged_results = JsonResults.merge(self._builder, aggregated_results, incremental_results, max_builds, sort_keys=True)
127
128         if expected_data:
129             expected_results = self._make_test_json(expected_data)
130             self.assertEquals(merged_results, expected_results)
131         else:
132             self.assertFalse(merged_results)
133
134     def _test_get_test_list(self, input_data, expected_data):
135         input_results = self._make_test_json(input_data)
136         expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("{[TESTDATA_TESTS]}", simplejson.dumps(expected_data, separators=(',', ':')))
137         actual_results = JsonResults.get_test_list(self._builder, input_results)
138         self.assertEquals(actual_results, expected_results)
139
140     def test_merge_null_incremental_results(self):
141         # Empty incremental results json.
142         # Nothing to merge.
143         self._test_merge(
144             # Aggregated results
145             {"builds": ["2", "1"],
146              "tests": {"001.html": {
147                            "results": [[200,"F"]],
148                            "times": [[200,0]]}}},
149             # Incremental results
150             None,
151             # Expect no merge happens.
152             None)
153
154     def test_merge_empty_incremental_results(self):
155         # No actual incremental test results (only prefix and suffix) to merge.
156         # Nothing to merge.
157         self._test_merge(
158             # Aggregated results
159             {"builds": ["2", "1"],
160              "tests": {"001.html": {
161                            "results": [[200,"F"]],
162                            "times": [[200,0]]}}},
163             # Incremental results
164             {"builds": [],
165              "tests": {}},
166             # Expected no merge happens.
167             None)
168
169     def test_merge_empty_aggregated_results(self):
170         # No existing aggregated results.
171         # Merged results == new incremental results.
172         self._test_merge(
173             # Aggregated results
174             None,
175             # Incremental results
176
177             {"builds": ["2", "1"],
178              "tests": {"001.html": {
179                            "results": [[200,"F"]],
180                            "times": [[200,0]]}}},
181             # Expected result
182             {"builds": ["2", "1"],
183              "tests": {"001.html": {
184                            "results": [[200,"F"]],
185                            "times": [[200,0]]}}})
186
187     def test_merge_incremental_single_test_single_run_same_result(self):
188         # Incremental results has the latest build and same test results for
189         # that run.
190         # Insert the incremental results at the first place and sum number
191         # of runs for "F" (200 + 1) to get merged results.
192         self._test_merge(
193             # Aggregated results
194             {"builds": ["2", "1"],
195              "tests": {"001.html": {
196                            "results": [[200,"F"]],
197                            "times": [[200,0]]}}},
198             # Incremental results
199             {"builds": ["3"],
200              "tests": {"001.html": {
201                            "results": [[1,"F"]],
202                            "times": [[1,0]]}}},
203             # Expected results
204             {"builds": ["3", "2", "1"],
205              "tests": {"001.html": {
206                            "results": [[201,"F"]],
207                            "times": [[201,0]]}}})
208
209     def test_merge_single_test_single_run_different_result(self):
210         # Incremental results has the latest build but different test results
211         # for that run.
212         # Insert the incremental results at the first place.
213         self._test_merge(
214             # Aggregated results
215             {"builds": ["2", "1"],
216              "tests": {"001.html": {
217                            "results": [[200,"F"]],
218                            "times": [[200,0]]}}},
219             # Incremental results
220             {"builds": ["3"],
221              "tests": {"001.html": {
222                            "results": [[1, "I"]],
223                            "times": [[1,1]]}}},
224             # Expected results
225             {"builds": ["3", "2", "1"],
226              "tests": {"001.html": {
227                            "results": [[1,"I"],[200,"F"]],
228                            "times": [[1,1],[200,0]]}}})
229
230     def test_merge_single_test_single_run_result_changed(self):
231         # Incremental results has the latest build but results which differ from
232         # the latest result (but are the same as an older result).
233         self._test_merge(
234             # Aggregated results
235             {"builds": ["2", "1"],
236              "tests": {"001.html": {
237                            "results": [[200,"F"],[10,"I"]],
238                            "times": [[200,0],[10,1]]}}},
239             # Incremental results
240             {"builds": ["3"],
241              "tests": {"001.html": {
242                            "results": [[1,"I"]],
243                            "times": [[1,1]]}}},
244             # Expected results
245             {"builds": ["3", "2", "1"],
246              "tests": {"001.html": {
247                            "results": [[1,"I"],[200,"F"],[10,"I"]],
248                            "times": [[1,1],[200,0],[10,1]]}}})
249
250     def test_merge_multiple_tests_single_run(self):
251         # All tests have incremental updates.
252         self._test_merge(
253             # Aggregated results
254             {"builds": ["2", "1"],
255              "tests": {"001.html": {
256                            "results": [[200,"F"]],
257                            "times": [[200,0]]},
258                        "002.html": {
259                            "results": [[100,"I"]],
260                            "times": [[100,1]]}}},
261             # Incremental results
262             {"builds": ["3"],
263              "tests": {"001.html": {
264                            "results": [[1,"F"]],
265                            "times": [[1,0]]},
266                        "002.html": {
267                            "results": [[1,"I"]],
268                            "times": [[1,1]]}}},
269             # Expected results
270             {"builds": ["3", "2", "1"],
271              "tests": {"001.html": {
272                            "results": [[201,"F"]],
273                            "times": [[201,0]]},
274                        "002.html": {
275                            "results": [[101,"I"]],
276                            "times": [[101,1]]}}})
277
278     def test_merge_multiple_tests_single_run_one_no_result(self):
279         self._test_merge(
280             # Aggregated results
281             {"builds": ["2", "1"],
282              "tests": {"001.html": {
283                            "results": [[200,"F"]],
284                            "times": [[200,0]]},
285                        "002.html": {
286                            "results": [[100,"I"]],
287                            "times": [[100,1]]}}},
288             # Incremental results
289             {"builds": ["3"],
290              "tests": {"002.html": {
291                            "results": [[1,"I"]],
292                            "times": [[1,1]]}}},
293             # Expected results
294             {"builds": ["3", "2", "1"],
295              "tests": {"001.html": {
296                            "results": [[1,"N"],[200,"F"]],
297                            "times": [[201,0]]},
298                        "002.html": {
299                            "results": [[101,"I"]],
300                            "times": [[101,1]]}}})
301
302     def test_merge_single_test_multiple_runs(self):
303         self._test_merge(
304             # Aggregated results
305             {"builds": ["2", "1"],
306              "tests": {"001.html": {
307                            "results": [[200,"F"]],
308                            "times": [[200,0]]}}},
309             # Incremental results
310             {"builds": ["4", "3"],
311              "tests": {"001.html": {
312                            "results": [[2, "I"]],
313                            "times": [[2,2]]}}},
314             # Expected results
315             {"builds": ["4", "3", "2", "1"],
316              "tests": {"001.html": {
317                            "results": [[2,"I"],[200,"F"]],
318                            "times": [[2,2],[200,0]]}}})
319
320     def test_merge_multiple_tests_multiple_runs(self):
321         self._test_merge(
322             # Aggregated results
323             {"builds": ["2", "1"],
324              "tests": {"001.html": {
325                            "results": [[200,"F"]],
326                            "times": [[200,0]]},
327                        "002.html": {
328                            "results": [[10,"Z"]],
329                            "times": [[10,0]]}}},
330             # Incremental results
331             {"builds": ["4", "3"],
332              "tests": {"001.html": {
333                            "results": [[2, "I"]],
334                            "times": [[2,2]]},
335                        "002.html": {
336                            "results": [[1,"C"]],
337                            "times": [[1,1]]}}},
338             # Expected results
339             {"builds": ["4", "3", "2", "1"],
340              "tests": {"001.html": {
341                            "results": [[2,"I"],[200,"F"]],
342                            "times": [[2,2],[200,0]]},
343                        "002.html": {
344                            "results": [[1,"C"],[10,"Z"]],
345                            "times": [[1,1],[10,0]]}}})
346
347     def test_merge_incremental_result_older_build(self):
348         # Test the build in incremental results is older than the most recent
349         # build in aggregated results.
350         self._test_merge(
351             # Aggregated results
352             {"builds": ["3", "1"],
353              "tests": {"001.html": {
354                            "results": [[5,"F"]],
355                            "times": [[5,0]]}}},
356             # Incremental results
357             {"builds": ["2"],
358              "tests": {"001.html": {
359                            "results": [[1, "F"]],
360                            "times": [[1,0]]}}},
361             # Expected no merge happens.
362             {"builds": ["2", "3", "1"],
363              "tests": {"001.html": {
364                            "results": [[6,"F"]],
365                            "times": [[6,0]]}}})
366
367     def test_merge_incremental_result_same_build(self):
368         # Test the build in incremental results is same as the build in
369         # aggregated results.
370         self._test_merge(
371             # Aggregated results
372             {"builds": ["2", "1"],
373              "tests": {"001.html": {
374                            "results": [[5,"F"]],
375                            "times": [[5,0]]}}},
376             # Incremental results
377             {"builds": ["3", "2"],
378              "tests": {"001.html": {
379                            "results": [[2, "F"]],
380                            "times": [[2,0]]}}},
381             # Expected no merge happens.
382             {"builds": ["3", "2", "2", "1"],
383              "tests": {"001.html": {
384                            "results": [[7,"F"]],
385                            "times": [[7,0]]}}})
386
387     def test_merge_remove_test_with_no_data(self):
388         # Remove test where there is no data in all runs.
389         self._test_merge(
390             # Aggregated results
391             {"builds": ["2", "1"],
392              "tests": {"001.html": {
393                            "results": [[200,"N"]],
394                            "times": [[200,0]]},
395                        "002.html": {
396                            "results": [[10,"F"]],
397                            "times": [[10,0]]}}},
398             # Incremental results
399             {"builds": ["3"],
400              "tests": {"001.html": {
401                            "results": [[1,"N"]],
402                            "times": [[1,0]]},
403                        "002.html": {
404                            "results": [[1,"P"]],
405                            "times": [[1,0]]}}},
406             # Expected results
407             {"builds": ["3", "2", "1"],
408              "tests": {"002.html": {
409                            "results": [[1,"P"],[10,"F"]],
410                            "times": [[11,0]]}}})
411
412     def test_merge_remove_test_with_all_pass(self):
413         # Remove test where all run pass and max running time < 1 seconds
414         self._test_merge(
415             # Aggregated results
416             {"builds": ["2", "1"],
417              "tests": {"001.html": {
418                            "results": [[200,"P"]],
419                            "times": [[200,0]]},
420                        "002.html": {
421                            "results": [[10,"F"]],
422                            "times": [[10,0]]}}},
423             # Incremental results
424             {"builds": ["3"],
425              "tests": {"001.html": {
426                            "results": [[1,"P"]],
427                            "times": [[1,0]]},
428                        "002.html": {
429                            "results": [[1,"P"]],
430                            "times": [[1,0]]}}},
431             # Expected results
432             {"builds": ["3", "2", "1"],
433              "tests": {"002.html": {
434                            "results": [[1,"P"],[10,"F"]],
435                            "times": [[11,0]]}}})
436
437     def test_merge_keep_test_with_all_pass_but_slow_time(self):
438         # Do not remove test where all run pass but max running time >= 1 seconds
439         self._test_merge(
440             # Aggregated results
441             {"builds": ["2", "1"],
442              "tests": {"001.html": {
443                            "results": [[200,"P"]],
444                            "times": [[200,0]]},
445                        "002.html": {
446                            "results": [[10,"F"]],
447                            "times": [[10,0]]}}},
448             # Incremental results
449             {"builds": ["3"],
450              "tests": {"001.html": {
451                            "results": [[1,"P"]],
452                            "times": [[1,1]]},
453                        "002.html": {
454                            "results": [[1,"P"]],
455                            "times": [[1,0]]}}},
456             # Expected results
457             {"builds": ["3", "2", "1"],
458              "tests": {"001.html": {
459                            "results": [[201,"P"]],
460                            "times": [[1,1],[200,0]]},
461                        "002.html": {
462                            "results": [[1,"P"],[10,"F"]],
463                            "times": [[11,0]]}}})
464
465     def test_merge_prune_extra_results(self):
466         # Remove items from test results and times that exceed the max number
467         # of builds to track.
468         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS
469         self._test_merge(
470             # Aggregated results
471             {"builds": ["2", "1"],
472              "tests": {"001.html": {
473                            "results": [[max_builds,"F"],[1,"I"]],
474                            "times": [[max_builds,0],[1,1]]}}},
475             # Incremental results
476             {"builds": ["3"],
477              "tests": {"001.html": {
478                            "results": [[1,"T"]],
479                            "times": [[1,1]]}}},
480             # Expected results
481             {"builds": ["3", "2", "1"],
482              "tests": {"001.html": {
483                            "results": [[1,"T"],[max_builds,"F"]],
484                            "times": [[1,1],[max_builds,0]]}}})
485
486     def test_merge_prune_extra_results_small(self):
487         # Remove items from test results and times that exceed the max number
488         # of builds to track, using smaller threshold.
489         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
490         self._test_merge(
491             # Aggregated results
492             {"builds": ["2", "1"],
493              "tests": {"001.html": {
494                            "results": [[max_builds,"F"],[1,"I"]],
495                            "times": [[max_builds,0],[1,1]]}}},
496             # Incremental results
497             {"builds": ["3"],
498              "tests": {"001.html": {
499                            "results": [[1,"T"]],
500                            "times": [[1,1]]}}},
501             # Expected results
502             {"builds": ["3", "2", "1"],
503              "tests": {"001.html": {
504                            "results": [[1,"T"],[max_builds,"F"]],
505                            "times": [[1,1],[max_builds,0]]}}},
506             int(max_builds))
507
508     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
509         # Test that merging in a new result of the same type as the last result
510         # causes old results to fall off.
511         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
512         self._test_merge(
513             # Aggregated results
514             {"builds": ["2", "1"],
515              "tests": {"001.html": {
516                            "results": [[max_builds,"F"],[1,"N"]],
517                            "times": [[max_builds,0],[1,1]]}}},
518             # Incremental results
519             {"builds": ["3"],
520              "tests": {"001.html": {
521                            "results": [[1,"F"]],
522                            "times": [[1,0]]}}},
523             # Expected results
524             {"builds": ["3", "2", "1"],
525              "tests": {"001.html": {
526                            "results": [[max_builds,"F"]],
527                            "times": [[max_builds,0]]}}},
528             int(max_builds))
529
530     def test_merge_build_directory_hierarchy_old_version(self):
531         self._test_merge(
532             # Aggregated results
533             {"builds": ["2", "1"],
534              "tests": {"bar/003.html": {
535                            "results": [[25,"F"]],
536                            "times": [[25,0]]},
537                        "foo/001.html": {
538                            "results": [[50,"F"]],
539                            "times": [[50,0]]},
540                        "foo/002.html": {
541                            "results": [[100,"I"]],
542                            "times": [[100,0]]}},
543              "version": 3},
544             # Incremental results
545             {"builds": ["3"],
546              "tests": {"baz": {
547                            "004.html": {
548                                "results": [[1,"I"]],
549                                "times": [[1,0]]}},
550                        "foo": {
551                            "001.html": {
552                                "results": [[1,"F"]],
553                                "times": [[1,0]]},
554                            "002.html": {
555                                "results": [[1,"I"]],
556                                "times": [[1,0]]}}},
557              "version": 4},
558             # Expected results
559             {"builds": ["3", "2", "1"],
560              "tests": {"bar": {
561                            "003.html": {
562                                "results": [[1,"N"],[25,"F"]],
563                                "times": [[26,0]]}},
564                        "baz": {
565                            "004.html": {
566                                "results": [[1,"I"]],
567                                "times": [[1,0]]}},
568                        "foo": {
569                            "001.html": {
570                                "results": [[51,"F"]],
571                                "times": [[51,0]]},
572                            "002.html": {
573                                "results": [[101,"I"]],
574                                "times": [[101,0]]}}},
575              "version": 4})
576
577     def test_merge_build_directory_hierarchy(self):
578         self._test_merge(
579             # Aggregated results
580             {"builds": ["2", "1"],
581              "tests": {"bar": {"baz": {
582                            "003.html": {
583                                 "results": [[25,"F"]],
584                                 "times": [[25,0]]}}},
585                        "foo": {
586                            "001.html": {
587                                 "results": [[50,"F"]],
588                                 "times": [[50,0]]},
589                            "002.html": {
590                                 "results": [[100,"I"]],
591                                 "times": [[100,0]]}}},
592               "version": 4},
593             # Incremental results
594             {"builds": ["3"],
595              "tests": {"baz": {
596                            "004.html": {
597                                "results": [[1,"I"]],
598                                "times": [[1,0]]}},
599                        "foo": {
600                            "001.html": {
601                                "results": [[1,"F"]],
602                                "times": [[1,0]]},
603                            "002.html": {
604                                "results": [[1,"I"]],
605                                "times": [[1,0]]}}},
606              "version": 4},
607             # Expected results
608             {"builds": ["3", "2", "1"],
609              "tests": {"bar": {"baz": {
610                            "003.html": {
611                                "results": [[1,"N"],[25,"F"]],
612                                "times": [[26,0]]}}},
613                        "baz": {
614                            "004.html": {
615                                "results": [[1,"I"]],
616                                "times": [[1,0]]}},
617                        "foo": {
618                            "001.html": {
619                                "results": [[51,"F"]],
620                                "times": [[51,0]]},
621                            "002.html": {
622                                "results": [[101,"I"]],
623                                "times": [[101,0]]}}},
624              "version": 4})
625
626     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
627
628     def test_get_test_name_list(self):
629         # Get test name list only. Don't include non-test-list data and
630         # of test result details.
631         # FIXME: This also tests a temporary bug in the data where directory-level
632         # results have a results and times values. Once that bug is fixed,
633         # remove this test-case and assert we don't ever hit it.
634         self._test_get_test_list(
635             # Input results
636             {"builds": ["3", "2", "1"],
637              "tests": {"foo": {
638                            "001.html": {
639                                "results": [[200,"P"]],
640                                "times": [[200,0]]},
641                            "results": [[1,"N"]],
642                            "times": [[1,0]]},
643                        "002.html": {
644                            "results": [[10,"F"]],
645                            "times": [[10,0]]}}},
646             # Expected results
647             {"foo": {"001.html":{}}, "002.html":{}})
648
649     def test_remove_gtest_modifiers(self):
650         self._test_merge(
651             # Aggregated results
652             {"builds": ["2", "1"],
653              "tests": {"foo.bar": {
654                            "results": [[50,"F"]],
655                            "times": [[50,0]]},
656                        "foo.bar2": {
657                            "results": [[100,"I"]],
658                            "times": [[100,0]]},
659                        "foo.FAILS_bar3": {
660                            "results": [[100,"I"]],
661                            "times": [[100,0]]},
662                        },
663              "version": 3},
664             # Incremental results
665             {"builds": ["3"],
666              "tests": {"foo.FLAKY_bar": {
667                            "results": [[1,"F"]],
668                            "times": [[1,0]]},
669                        "foo.DISABLED_bar2": {
670                            "results": [[1,"I"]],
671                            "times": [[1,0]]},
672                        "foo.bar3": {
673                            "results": [[1,"I"]],
674                            "times": [[1,0]]},
675                        "foo.FAILS_bar3": {
676                            "results": [[1,"I"]],
677                            "times": [[1,0]]},
678                        "foo.MAYBE_bar4": {
679                            "results": [[1,"I"]],
680                            "times": [[1,0]]}},
681              "version": 4},
682             # Expected results
683             {"builds": ["3", "2", "1"],
684              "tests": {"foo.bar": {
685                            "results": [[51,"F"]],
686                            "times": [[51,0]]},
687                        "foo.bar2": {
688                            "results": [[101,"I"]],
689                            "times": [[101,0]]},
690                        "foo.bar3": {
691                            "results": [[1,"I"]],
692                            "times": [[1,0]]},
693                        "foo.FAILS_bar3": {
694                               "results": [[1,"N"],[100,"I"]],
695                               "times": [[101,0]]},
696                        "foo.bar4": {
697                            "results": [[1,"I"]],
698                            "times": [[1,0]]}},
699              "version": 4})
700
701 if __name__ == '__main__':
702     unittest.main()