99e709a35fbe7448550f861b8303983843e35023
[WebKit-https.git] / Tools / TestResultServer / model / jsonresults_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 try:
30     import jsonresults
31     from jsonresults import JsonResults
32 except ImportError:
33     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
34     raise
35
36 import unittest
37
38
39 JSON_RESULTS_TEMPLATE = (
40     '{"Webkit":{'
41     '"allFixableCount":[[TESTDATA_COUNT]],'
42     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
43     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
44     '"deferredCounts":[[TESTDATA_COUNTS]],'
45     '"fixableCount":[[TESTDATA_COUNT]],'
46     '"fixableCounts":[[TESTDATA_COUNTS]],'
47     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
48     '"tests":{[TESTDATA_TESTS]},'
49     '"webkitRevision":[[TESTDATA_WEBKITREVISION]],'
50     '"wontfixCounts":[[TESTDATA_COUNTS]]'
51     '},'
52     '"version":[VERSION]'
53     '}')
54
55 JSON_RESULTS_COUNTS_TEMPLATE = (
56     '{'
57     '"C":[TESTDATA],'
58     '"F":[TESTDATA],'
59     '"I":[TESTDATA],'
60     '"O":[TESTDATA],'
61     '"P":[TESTDATA],'
62     '"T":[TESTDATA],'
63     '"X":[TESTDATA],'
64     '"Z":[TESTDATA]}')
65
66 JSON_RESULTS_DIRECTORY_TEMPLATE = '"[TESTDATA_DIRECTORY]":{[TESTDATA_DATA]}'
67
68 JSON_RESULTS_TESTS_TEMPLATE = (
69     '"[TESTDATA_TEST_NAME]":{'
70     '"results":[[TESTDATA_TEST_RESULTS]],'
71     '"times":[[TESTDATA_TEST_TIMES]]}')
72
73 JSON_RESULTS_TEST_LIST_TEMPLATE = (
74     '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}')
75
76
77 class JsonResultsTest(unittest.TestCase):
78     def setUp(self):
79         self._builder = "Webkit"
80
81     def test_strip_prefix_suffix(self):
82         json = "['contents']"
83         self.assertEqual(JsonResults._strip_prefix_suffix("ADD_RESULTS(" + json + ");"), json)
84         self.assertEqual(JsonResults._strip_prefix_suffix(json), json)
85
86     def _make_test_json(self, test_data):
87         if not test_data:
88             return ""
89
90         builds = test_data["builds"]
91         tests = test_data["tests"]
92         if not builds or not tests:
93             return ""
94
95         json = JSON_RESULTS_TEMPLATE
96
97         counts = []
98         build_numbers = []
99         webkit_revision = []
100         chrome_revision = []
101         times = []
102         for build in builds:
103             counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
104             build_numbers.append("1000%s" % build)
105             webkit_revision.append("2000%s" % build)
106             chrome_revision.append("3000%s" % build)
107             times.append("100000%s000" % build)
108
109         json = json.replace("[TESTDATA_COUNTS]", ",".join(counts))
110         json = json.replace("[TESTDATA_COUNT]", ",".join(builds))
111         json = json.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
112         json = json.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
113         json = json.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
114         json = json.replace("[TESTDATA_TIMES]", ",".join(times))
115
116         version = str(test_data["version"]) if "version" in test_data else "4"
117         json = json.replace("[VERSION]", version)
118
119         json_tests = []
120         for (name, test) in sorted(tests.iteritems()):
121             json_tests.append(self._parse_tests_dict(name, test))
122
123         json = json.replace("[TESTDATA_TESTS]", ",".join(json_tests))
124
125         return json
126
127     def _parse_tests_dict(self, name, test):
128         if "results" in test:
129             test_results = JSON_RESULTS_TESTS_TEMPLATE.replace("[TESTDATA_TEST_NAME]", name)
130             test_results = test_results.replace("[TESTDATA_TEST_RESULTS]", test["results"])
131             test_results = test_results.replace("[TESTDATA_TEST_TIMES]", test["times"])
132             return test_results
133
134         test_results = JSON_RESULTS_DIRECTORY_TEMPLATE.replace("[TESTDATA_DIRECTORY]", name)
135         testdata = []
136         for (child_name, child_test) in sorted(test.iteritems()):
137             testdata.append(self._parse_tests_dict(child_name, child_test))
138         test_results = test_results.replace("[TESTDATA_DATA]", ",".join(testdata))
139         return test_results
140
141     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
142         aggregated_results = self._make_test_json(aggregated_data)
143         incremental_results = self._make_test_json(incremental_data)
144         merged_results = JsonResults.merge(self._builder, aggregated_results, incremental_results, max_builds, sort_keys=True)
145
146         if expected_data:
147             expected_results = self._make_test_json(expected_data)
148             self.assertEquals(merged_results, expected_results)
149         else:
150             self.assertFalse(merged_results)
151
152     def _test_get_test_list(self, input_data, expected_data):
153         input_results = self._make_test_json(input_data)
154
155         json_tests = []
156         for test in expected_data:
157             json_tests.append("\"" + test + "\":{}")
158
159         expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("[TESTDATA_TESTS]", ",".join(json_tests))
160         actual_results = JsonResults.get_test_list(self._builder, input_results)
161         self.assertEquals(actual_results, expected_results)
162
163     def test_merge_null_incremental_results(self):
164         # Empty incremental results json.
165         # Nothing to merge.
166         self._test_merge(
167             # Aggregated results
168             {"builds": ["2", "1"],
169              "tests": {"001.html": {
170                            "results": "[200,\"F\"]",
171                            "times": "[200,0]"}}},
172             # Incremental results
173             None,
174             # Expect no merge happens.
175             None)
176
177     def test_merge_empty_incremental_results(self):
178         # No actual incremental test results (only prefix and suffix) to merge.
179         # Nothing to merge.
180         self._test_merge(
181             # Aggregated results
182             {"builds": ["2", "1"],
183              "tests": {"001.html": {
184                            "results": "[200,\"F\"]",
185                            "times": "[200,0]"}}},
186             # Incremental results
187             {"builds": [],
188              "tests": {}},
189             # Expected no merge happens.
190             None)
191
192     def test_merge_empty_aggregated_results(self):
193         # No existing aggregated results.
194         # Merged results == new incremental results.
195         self._test_merge(
196             # Aggregated results
197             None,
198             # Incremental results
199
200             {"builds": ["2", "1"],
201              "tests": {"001.html": {
202                            "results": "[200,\"F\"]",
203                            "times": "[200,0]"}}},
204             # Expected result
205             {"builds": ["2", "1"],
206              "tests": {"001.html": {
207                            "results": "[200,\"F\"]",
208                            "times": "[200,0]"}}})
209
210     def test_merge_incremental_single_test_single_run_same_result(self):
211         # Incremental results has the latest build and same test results for
212         # that run.
213         # Insert the incremental results at the first place and sum number
214         # of runs for "F" (200 + 1) to get merged results.
215         self._test_merge(
216             # Aggregated results
217             {"builds": ["2", "1"],
218              "tests": {"001.html": {
219                            "results": "[200,\"F\"]",
220                            "times": "[200,0]"}}},
221             # Incremental results
222             {"builds": ["3"],
223              "tests": {"001.html": {
224                            "results": "[1,\"F\"]",
225                            "times": "[1,0]"}}},
226             # Expected results
227             {"builds": ["3", "2", "1"],
228              "tests": {"001.html": {
229                            "results": "[201,\"F\"]",
230                            "times": "[201,0]"}}})
231
232     def test_merge_single_test_single_run_different_result(self):
233         # Incremental results has the latest build but different test results
234         # for that run.
235         # Insert the incremental results at the first place.
236         self._test_merge(
237             # Aggregated results
238             {"builds": ["2", "1"],
239              "tests": {"001.html": {
240                            "results": "[200,\"F\"]",
241                            "times": "[200,0]"}}},
242             # Incremental results
243             {"builds": ["3"],
244              "tests": {"001.html": {
245                            "results": "[1, \"I\"]",
246                            "times": "[1,1]"}}},
247             # Expected results
248             {"builds": ["3", "2", "1"],
249              "tests": {"001.html": {
250                            "results": "[1,\"I\"],[200,\"F\"]",
251                            "times": "[1,1],[200,0]"}}})
252
253     def test_merge_single_test_single_run_result_changed(self):
254         # Incremental results has the latest build but results which differ from
255         # the latest result (but are the same as an older result).
256         self._test_merge(
257             # Aggregated results
258             {"builds": ["2", "1"],
259              "tests": {"001.html": {
260                            "results": "[200,\"F\"],[10,\"I\"]",
261                            "times": "[200,0],[10,1]"}}},
262             # Incremental results
263             {"builds": ["3"],
264              "tests": {"001.html": {
265                            "results": "[1,\"I\"]",
266                            "times": "[1,1]"}}},
267             # Expected results
268             {"builds": ["3", "2", "1"],
269              "tests": {"001.html": {
270                            "results": "[1,\"I\"],[200,\"F\"],[10,\"I\"]",
271                            "times": "[1,1],[200,0],[10,1]"}}})
272
273     def test_merge_multiple_tests_single_run(self):
274         # All tests have incremental updates.
275         self._test_merge(
276             # Aggregated results
277             {"builds": ["2", "1"],
278              "tests": {"001.html": {
279                            "results": "[200,\"F\"]",
280                            "times": "[200,0]"},
281                        "002.html": {
282                            "results": "[100,\"I\"]",
283                            "times": "[100,1]"}}},
284             # Incremental results
285             {"builds": ["3"],
286              "tests": {"001.html": {
287                            "results": "[1,\"F\"]",
288                            "times": "[1,0]"},
289                        "002.html": {
290                            "results": "[1,\"I\"]",
291                            "times": "[1,1]"}}},
292             # Expected results
293             {"builds": ["3", "2", "1"],
294              "tests": {"001.html": {
295                            "results": "[201,\"F\"]",
296                            "times": "[201,0]"},
297                        "002.html": {
298                            "results": "[101,\"I\"]",
299                            "times": "[101,1]"}}})
300
301     def test_merge_multiple_tests_single_run_one_no_result(self):
302         self._test_merge(
303             # Aggregated results
304             {"builds": ["2", "1"],
305              "tests": {"001.html": {
306                            "results": "[200,\"F\"]",
307                            "times": "[200,0]"},
308                        "002.html": {
309                            "results": "[100,\"I\"]",
310                            "times": "[100,1]"}}},
311             # Incremental results
312             {"builds": ["3"],
313              "tests": {"002.html": {
314                            "results": "[1,\"I\"]",
315                            "times": "[1,1]"}}},
316             # Expected results
317             {"builds": ["3", "2", "1"],
318              "tests": {"001.html": {
319                            "results": "[1,\"N\"],[200,\"F\"]",
320                            "times": "[201,0]"},
321                        "002.html": {
322                            "results": "[101,\"I\"]",
323                            "times": "[101,1]"}}})
324
325     def test_merge_single_test_multiple_runs(self):
326         self._test_merge(
327             # Aggregated results
328             {"builds": ["2", "1"],
329              "tests": {"001.html": {
330                            "results": "[200,\"F\"]",
331                            "times": "[200,0]"}}},
332             # Incremental results
333             {"builds": ["4", "3"],
334              "tests": {"001.html": {
335                            "results": "[2, \"I\"]",
336                            "times": "[2,2]"}}},
337             # Expected results
338             {"builds": ["4", "3", "2", "1"],
339              "tests": {"001.html": {
340                            "results": "[2,\"I\"],[200,\"F\"]",
341                            "times": "[2,2],[200,0]"}}})
342
343     def test_merge_multiple_tests_multiple_runs(self):
344         self._test_merge(
345             # Aggregated results
346             {"builds": ["2", "1"],
347              "tests": {"001.html": {
348                            "results": "[200,\"F\"]",
349                            "times": "[200,0]"},
350                        "002.html": {
351                            "results": "[10,\"Z\"]",
352                            "times": "[10,0]"}}},
353             # Incremental results
354             {"builds": ["4", "3"],
355              "tests": {"001.html": {
356                            "results": "[2, \"I\"]",
357                            "times": "[2,2]"},
358                        "002.html": {
359                            "results": "[1,\"C\"]",
360                            "times": "[1,1]"}}},
361             # Expected results
362             {"builds": ["4", "3", "2", "1"],
363              "tests": {"001.html": {
364                            "results": "[2,\"I\"],[200,\"F\"]",
365                            "times": "[2,2],[200,0]"},
366                        "002.html": {
367                            "results": "[1,\"C\"],[10,\"Z\"]",
368                            "times": "[1,1],[10,0]"}}})
369
370     def test_merge_incremental_result_older_build(self):
371         # Test the build in incremental results is older than the most recent
372         # build in aggregated results.
373         self._test_merge(
374             # Aggregated results
375             {"builds": ["3", "1"],
376              "tests": {"001.html": {
377                            "results": "[5,\"F\"]",
378                            "times": "[5,0]"}}},
379             # Incremental results
380             {"builds": ["2"],
381              "tests": {"001.html": {
382                            "results": "[1, \"F\"]",
383                            "times": "[1,0]"}}},
384             # Expected no merge happens.
385             {"builds": ["2", "3", "1"],
386              "tests": {"001.html": {
387                            "results": "[6,\"F\"]",
388                            "times": "[6,0]"}}})
389
390     def test_merge_incremental_result_same_build(self):
391         # Test the build in incremental results is same as the build in
392         # aggregated results.
393         self._test_merge(
394             # Aggregated results
395             {"builds": ["2", "1"],
396              "tests": {"001.html": {
397                            "results": "[5,\"F\"]",
398                            "times": "[5,0]"}}},
399             # Incremental results
400             {"builds": ["3", "2"],
401              "tests": {"001.html": {
402                            "results": "[2, \"F\"]",
403                            "times": "[2,0]"}}},
404             # Expected no merge happens.
405             {"builds": ["3", "2", "2", "1"],
406              "tests": {"001.html": {
407                            "results": "[7,\"F\"]",
408                            "times": "[7,0]"}}})
409
410     def test_merge_remove_test_with_no_data(self):
411         # Remove test where there is no data in all runs.
412         self._test_merge(
413             # Aggregated results
414             {"builds": ["2", "1"],
415              "tests": {"001.html": {
416                            "results": "[200,\"N\"]",
417                            "times": "[200,0]"},
418                        "002.html": {
419                            "results": "[10,\"F\"]",
420                            "times": "[10,0]"}}},
421             # Incremental results
422             {"builds": ["3"],
423              "tests": {"001.html": {
424                            "results": "[1,\"N\"]",
425                            "times": "[1,0]"},
426                        "002.html": {
427                            "results": "[1,\"P\"]",
428                            "times": "[1,0]"}}},
429             # Expected results
430             {"builds": ["3", "2", "1"],
431              "tests": {"002.html": {
432                            "results": "[1,\"P\"],[10,\"F\"]",
433                            "times": "[11,0]"}}})
434
435     def test_merge_remove_test_with_all_pass(self):
436         # Remove test where all run pass and max running time < 1 seconds
437         self._test_merge(
438             # Aggregated results
439             {"builds": ["2", "1"],
440              "tests": {"001.html": {
441                            "results": "[200,\"P\"]",
442                            "times": "[200,0]"},
443                        "002.html": {
444                            "results": "[10,\"F\"]",
445                            "times": "[10,0]"}}},
446             # Incremental results
447             {"builds": ["3"],
448              "tests": {"001.html": {
449                            "results": "[1,\"P\"]",
450                            "times": "[1,0]"},
451                        "002.html": {
452                            "results": "[1,\"P\"]",
453                            "times": "[1,0]"}}},
454             # Expected results
455             {"builds": ["3", "2", "1"],
456              "tests": {"002.html": {
457                            "results": "[1,\"P\"],[10,\"F\"]",
458                            "times": "[11,0]"}}})
459
460     def test_merge_keep_test_with_all_pass_but_slow_time(self):
461         # Do not remove test where all run pass but max running time >= 1 seconds
462         self._test_merge(
463             # Aggregated results
464             {"builds": ["2", "1"],
465              "tests": {"001.html": {
466                            "results": "[200,\"P\"]",
467                            "times": "[200,0]"},
468                        "002.html": {
469                            "results": "[10,\"F\"]",
470                            "times": "[10,0]"}}},
471             # Incremental results
472             {"builds": ["3"],
473              "tests": {"001.html": {
474                            "results": "[1,\"P\"]",
475                            "times": "[1,1]"},
476                        "002.html": {
477                            "results": "[1,\"P\"]",
478                            "times": "[1,0]"}}},
479             # Expected results
480             {"builds": ["3", "2", "1"],
481              "tests": {"001.html": {
482                            "results": "[201,\"P\"]",
483                            "times": "[1,1],[200,0]"},
484                        "002.html": {
485                            "results": "[1,\"P\"],[10,\"F\"]",
486                            "times": "[11,0]"}}})
487
488     def test_merge_prune_extra_results(self):
489         # Remove items from test results and times that exceed the max number
490         # of builds to track.
491         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS)
492         self._test_merge(
493             # Aggregated results
494             {"builds": ["2", "1"],
495              "tests": {"001.html": {
496                            "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
497                            "times": "[" + max_builds + ",0],[1,1]"}}},
498             # Incremental results
499             {"builds": ["3"],
500              "tests": {"001.html": {
501                            "results": "[1,\"T\"]",
502                            "times": "[1,1]"}}},
503             # Expected results
504             {"builds": ["3", "2", "1"],
505              "tests": {"001.html": {
506                            "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
507                            "times": "[1,1],[" + max_builds + ",0]"}}})
508
509     def test_merge_prune_extra_results_small(self):
510         # Remove items from test results and times that exceed the max number
511         # of builds to track, using smaller threshold.
512         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
513         self._test_merge(
514             # Aggregated results
515             {"builds": ["2", "1"],
516              "tests": {"001.html": {
517                            "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
518                            "times": "[" + max_builds + ",0],[1,1]"}}},
519             # Incremental results
520             {"builds": ["3"],
521              "tests": {"001.html": {
522                            "results": "[1,\"T\"]",
523                            "times": "[1,1]"}}},
524             # Expected results
525             {"builds": ["3", "2", "1"],
526              "tests": {"001.html": {
527                            "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
528                            "times": "[1,1],[" + max_builds + ",0]"}}},
529             int(max_builds))
530
531     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
532         # Test that merging in a new result of the same type as the last result
533         # causes old results to fall off.
534         max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
535         self._test_merge(
536             # Aggregated results
537             {"builds": ["2", "1"],
538              "tests": {"001.html": {
539                            "results": "[" + max_builds + ",\"F\"],[1,\"N\"]",
540                            "times": "[" + max_builds + ",0],[1,1]"}}},
541             # Incremental results
542             {"builds": ["3"],
543              "tests": {"001.html": {
544                            "results": "[1,\"F\"]",
545                            "times": "[1,0]"}}},
546             # Expected results
547             {"builds": ["3", "2", "1"],
548              "tests": {"001.html": {
549                            "results": "[" + max_builds + ",\"F\"]",
550                            "times": "[" + max_builds + ",0]"}}},
551             int(max_builds))
552
553     def test_merge_build_directory_hierarchy_old_version(self):
554         self._test_merge(
555             # Aggregated results
556             {"builds": ["2", "1"],
557              "tests": {"bar/003.html": {
558                            "results": "[25,\"F\"]",
559                            "times": "[25,0]"},
560                        "foo/001.html": {
561                            "results": "[50,\"F\"]",
562                            "times": "[50,0]"},
563                        "foo/002.html": {
564                            "results": "[100,\"I\"]",
565                            "times": "[100,0]"}},
566              "version": 3},
567             # Incremental results
568             {"builds": ["3"],
569              "tests": {"baz": {
570                            "004.html": {
571                                "results": "[1,\"I\"]",
572                                "times": "[1,0]"}},
573                        "foo": {
574                            "001.html": {
575                                "results": "[1,\"F\"]",
576                                "times": "[1,0]"},
577                            "002.html": {
578                                "results": "[1,\"I\"]",
579                                "times": "[1,0]"}}},
580              "version": 4},
581             # Expected results
582             {"builds": ["3", "2", "1"],
583              "tests": {"bar": {
584                            "003.html": {
585                                "results": "[1,\"N\"],[25,\"F\"]",
586                                "times": "[26,0]"}},
587                        "baz": {
588                            "004.html": {
589                                "results": "[1,\"I\"]",
590                                "times": "[1,0]"}},
591                        "foo": {
592                            "001.html": {
593                                "results": "[51,\"F\"]",
594                                "times": "[51,0]"},
595                            "002.html": {
596                                "results": "[101,\"I\"]",
597                                "times": "[101,0]"}}},
598              "version": 4})
599
600     def test_merge_build_directory_hierarchy(self):
601         self._test_merge(
602             # Aggregated results
603             {"builds": ["2", "1"],
604              "tests": {"bar": {"baz": {
605                            "003.html": {
606                                 "results": "[25,\"F\"]",
607                                 "times": "[25,0]"}}},
608                        "foo": {
609                            "001.html": {
610                                 "results": "[50,\"F\"]",
611                                 "times": "[50,0]"},
612                            "002.html": {
613                                 "results": "[100,\"I\"]",
614                                 "times": "[100,0]"}}},
615               "version": 4},
616             # Incremental results
617             {"builds": ["3"],
618              "tests": {"baz": {
619                            "004.html": {
620                                "results": "[1,\"I\"]",
621                                "times": "[1,0]"}},
622                        "foo": {
623                            "001.html": {
624                                "results": "[1,\"F\"]",
625                                "times": "[1,0]"},
626                            "002.html": {
627                                "results": "[1,\"I\"]",
628                                "times": "[1,0]"}}},
629              "version": 4},
630             # Expected results
631             {"builds": ["3", "2", "1"],
632              "tests": {"bar": {"baz": {
633                            "003.html": {
634                                "results": "[1,\"N\"],[25,\"F\"]",
635                                "times": "[26,0]"}}},
636                        "baz": {
637                            "004.html": {
638                                "results": "[1,\"I\"]",
639                                "times": "[1,0]"}},
640                        "foo": {
641                            "001.html": {
642                                "results": "[51,\"F\"]",
643                                "times": "[51,0]"},
644                            "002.html": {
645                                "results": "[101,\"I\"]",
646                                "times": "[101,0]"}}},
647              "version": 4})
648
649     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
650
651     def test_get_test_name_list(self):
652         # Get test name list only. Don't include non-test-list data and
653         # of test result details.
654         self._test_get_test_list(
655             # Input results
656             {"builds": ["3", "2", "1"],
657              "tests": {"001.html": {
658                            "results": "[200,\"P\"]",
659                            "times": "[200,0]"},
660                        "002.html": {
661                            "results": "[10,\"F\"]",
662                            "times": "[10,0]"}}},
663             # Expected results
664             ["001.html", "002.html"])
665
666     def test_remove_gtest_modifiers(self):
667         self._test_merge(
668             # Aggregated results
669             {"builds": ["2", "1"],
670              "tests": {"foo.bar": {
671                            "results": "[50,\"F\"]",
672                            "times": "[50,0]"},
673                        "foo.bar2": {
674                            "results": "[100,\"I\"]",
675                            "times": "[100,0]"},
676                        "foo.FAILS_bar3": {
677                            "results": "[100,\"I\"]",
678                            "times": "[100,0]"},
679                        },
680              "version": 3},
681             # Incremental results
682             {"builds": ["3"],
683              "tests": {"foo.FLAKY_bar": {
684                            "results": "[1,\"F\"]",
685                            "times": "[1,0]"},
686                        "foo.DISABLED_bar2": {
687                            "results": "[1,\"I\"]",
688                            "times": "[1,0]"},
689                        "foo.bar3": {
690                            "results": "[1,\"I\"]",
691                            "times": "[1,0]"},
692                        "foo.FAILS_bar3": {
693                            "results": "[1,\"I\"]",
694                            "times": "[1,0]"},
695                        "foo.MAYBE_bar4": {
696                            "results": "[1,\"I\"]",
697                            "times": "[1,0]"}},
698              "version": 4},
699             # Expected results
700             {"builds": ["3", "2", "1"],
701              "tests": {"foo.bar": {
702                            "results": "[51,\"F\"]",
703                            "times": "[51,0]"},
704                        "foo.bar2": {
705                            "results": "[101,\"I\"]",
706                            "times": "[101,0]"},
707                        "foo.bar3": {
708                            "results": "[1,\"I\"]",
709                            "times": "[1,0]"},
710                        "foo.FAILS_bar3": {
711                               "results": "[1,\"N\"],[100,\"I\"]",
712                               "times": "[101,0]"},
713                        "foo.bar4": {
714                            "results": "[1,\"I\"]",
715                            "times": "[1,0]"}},
716              "version": 4})
717
718 if __name__ == '__main__':
719     unittest.main()