run-webkit-tests prints confusing messages when test expectations list results that...
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / views / buildbot_results_unittest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import StringIO
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.tool.mocktool import MockOptions
34
35 from webkitpy.layout_tests.models import test_expectations
36 from webkitpy.layout_tests.models import test_failures
37 from webkitpy.layout_tests.models import test_run_results
38 from webkitpy.layout_tests.models import test_run_results
39 from webkitpy.layout_tests.models import test_run_results_unittest
40 from webkitpy.layout_tests.views import buildbot_results
41
42
43 class BuildBotPrinterTests(unittest.TestCase):
44     def assertEmpty(self, stream):
45         self.assertFalse(stream.getvalue())
46
47     def assertNotEmpty(self, stream):
48         self.assertTrue(stream.getvalue())
49
50     def get_printer(self):
51         stream = StringIO.StringIO()
52         printer = buildbot_results.BuildBotPrinter(stream, debug_logging=True)
53         return printer, stream
54
55     def test_print_unexpected_results(self):
56         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
57         printer, out = self.get_printer()
58
59         # test everything running as expected
60         DASHED_LINE = "-" * 78 + "\n"
61         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
62         printer.print_unexpected_results(summary)
63         self.assertEqual(out.getvalue(), DASHED_LINE)
64
65         # test failures
66         printer, out = self.get_printer()
67         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
68         printer.print_unexpected_results(summary)
69         self.assertNotEmpty(out)
70
71         # test unexpected flaky
72         printer, out = self.get_printer()
73         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True)
74         printer.print_unexpected_results(summary)
75         self.assertNotEmpty(out)
76
77         printer, out = self.get_printer()
78         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
79         printer.print_unexpected_results(summary)
80         self.assertNotEmpty(out)
81
82         printer, out = self.get_printer()
83         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
84         printer.print_unexpected_results(summary)
85         self.assertNotEmpty(out)
86
87         printer, out = self.get_printer()
88         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
89         printer.print_unexpected_results(summary)
90         self.assertNotEmpty(out)
91
92     def test_print_unexpected_results_with_options(self):
93         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
94
95         DASHED_LINE = "-" * 78 + "\n"
96
97         port._options.pixel_tests = True
98         port._options.world_leaks = False
99         printer, out = self.get_printer()
100         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
101         printer.print_unexpected_results(summary)
102         self.assertEqual(out.getvalue(), DASHED_LINE)
103
104         port._options.pixel_tests = False
105         port._options.world_leaks = True
106         printer, out = self.get_printer()
107         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
108         printer.print_unexpected_results(summary)
109         self.assertEqual(out.getvalue(), DASHED_LINE)
110
111         port._options.pixel_tests = True
112         port._options.world_leaks = True
113         printer, out = self.get_printer()
114         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
115         printer.print_unexpected_results(summary)
116         self.assertEqual(out.getvalue(), DASHED_LINE)
117
118     def test_print_results(self):
119         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
120         printer, out = self.get_printer()
121         initial_results = test_run_results_unittest.run_results(port)
122         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
123         details = test_run_results.RunDetails(summary['num_regressions'], summary, initial_results, None)
124         printer.print_results(details)
125         self.assertNotEmpty(out)