check-webkit-style of the chromium test_expectations.txt file doesn't test all chromi...
[WebKit-https.git] / Tools / Scripts / webkitpy / style / checkers / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import os
31 import sys
32 import unittest
33
34 from test_expectations import TestExpectationsChecker
35 from webkitpy.common.host_mock import MockHost
36
37
38 class ErrorCollector(object):
39     """An error handler class for unit tests."""
40
41     def __init__(self):
42         self._errors = []
43         self.turned_off_filtering = False
44
45     def turn_off_line_filtering(self):
46         self.turned_off_filtering = True
47
48     def __call__(self, lineno, category, confidence, message):
49         self._errors.append('%s  [%s] [%d]' % (message, category, confidence))
50
51     def get_errors(self):
52         return ''.join(self._errors)
53
54     def reset_errors(self):
55         self._errors = []
56         self.turned_off_filtering = False
57
58
59 class TestExpectationsTestCase(unittest.TestCase):
60     """TestCase for test_expectations.py"""
61
62     def setUp(self):
63         self._error_collector = ErrorCollector()
64         self._test_file = 'passes/text.html'
65
66     def _expect_port_for_expectations_path(self, expected_port_or_port_class, expectations_path):
67         host = MockHost()
68         checker = TestExpectationsChecker(expectations_path, ErrorCollector(), host=host)
69         port = checker._determine_port_from_exepectations_path(host, expectations_path)
70         if port:
71             self.assertEquals(port.__class__.__name__, expected_port_or_port_class)
72         else:
73             self.assertEquals(port, expected_port_or_port_class)
74
75     def test_determine_port_from_exepectations_path(self):
76         self._expect_port_for_expectations_path(None, "/")
77         self._expect_port_for_expectations_path("ChromiumMacPort", "/mock-checkout/LayoutTests/chromium-mac/test_expectations.txt")
78
79     def test_check_covers_all_configurations(self):
80         checker = TestExpectationsChecker('test/test_expectations.txt', self._error_collector, host=MockHost())
81         output = []
82
83         def mock_check_test_expectations(expectations_str, test_configuration, tests, overrides, output=output):
84             output.append(str(test_configuration))
85         checker.check_test_expectations = mock_check_test_expectations
86         checker.check(lines="")
87
88         expected_output = """<leopard, x86, debug, cpu>
89 <leopard, x86, debug, gpu>
90 <leopard, x86, release, cpu>
91 <leopard, x86, release, gpu>
92 <snowleopard, x86, debug, cpu>
93 <snowleopard, x86, debug, gpu>
94 <snowleopard, x86, release, cpu>
95 <snowleopard, x86, release, gpu>
96 <xp, x86, debug, cpu>
97 <xp, x86, debug, gpu>
98 <xp, x86, release, cpu>
99 <xp, x86, release, gpu>
100 <vista, x86, debug, cpu>
101 <vista, x86, debug, gpu>
102 <vista, x86, release, cpu>
103 <vista, x86, release, gpu>
104 <win7, x86, debug, cpu>
105 <win7, x86, debug, gpu>
106 <win7, x86, release, cpu>
107 <win7, x86, release, gpu>
108 <lucid, x86, debug, cpu>
109 <lucid, x86, debug, gpu>
110 <lucid, x86, release, cpu>
111 <lucid, x86, release, gpu>
112 <lucid, x86_64, debug, cpu>
113 <lucid, x86_64, debug, gpu>
114 <lucid, x86_64, release, cpu>
115 <lucid, x86_64, release, gpu>"""
116
117         self.assertEqual("\n".join(output), expected_output)
118
119     def assert_lines_lint(self, lines, expected):
120         self._error_collector.reset_errors()
121         checker = TestExpectationsChecker('test/test_expectations.txt',
122                                           self._error_collector, host=MockHost())
123         checker.check_test_expectations(expectations_str='\n'.join(lines),
124                                         test_configuration=checker._port_obj.test_configuration(),
125                                         tests=[self._test_file],
126                                         overrides=None)
127         checker.check_tabs(lines)
128         self.assertEqual(expected, self._error_collector.get_errors())
129         self.assertTrue(self._error_collector.turned_off_filtering)
130
131     def test_valid_expectations(self):
132         self.assert_lines_lint(
133             ["BUGCR1234 MAC : passes/text.html = PASS FAIL"],
134             "")
135         self.assert_lines_lint(
136             ["SKIP BUGCR1234 : passes/text.html = TIMEOUT PASS"],
137             "")
138         self.assert_lines_lint(
139             ["BUGCR1234 DEBUG : passes/text.html = TIMEOUT PASS"],
140             "")
141         self.assert_lines_lint(
142             ["BUGCR1234 DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
143             "")
144         self.assert_lines_lint(
145             ["BUGCR1234 MAC DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
146             "")
147         self.assert_lines_lint(
148             ["BUGCR1234 DEBUG MAC : passes/text.html = TIMEOUT PASS"],
149             "")
150         self.assert_lines_lint(
151             ["SLOW BUGCR1234 : passes/text.html = PASS"],
152             "")
153         self.assert_lines_lint(
154             ["WONTFIX SKIP : passes/text.html = TIMEOUT"],
155             "")
156
157     def test_modifier_errors(self):
158         self.assert_lines_lint(
159             ["BUG1234 : passes/text.html = FAIL"],
160             "BUG\\d+ is not allowed, must be one of BUGCR\\d+, BUGWK\\d+, BUGV8_\\d+, or a non-numeric bug identifier. passes/text.html  [test/expectations] [5]")
161
162     def test_valid_modifiers(self):
163         self.assert_lines_lint(
164             ["INVALID-MODIFIER : passes/text.html = PASS"],
165             "Unrecognized modifier 'invalid-modifier' "
166             "passes/text.html  [test/expectations] [5]")
167         self.assert_lines_lint(
168             ["SKIP : passes/text.html = PASS"],
169             "Test lacks BUG modifier. "
170             "passes/text.html  [test/expectations] [2]")
171
172     def test_expectation_errors(self):
173         self.assert_lines_lint(
174             ["missing expectations"],
175             "Missing a ':' missing expectations  [test/expectations] [5]")
176         self.assert_lines_lint(
177             ["SLOW : passes/text.html = TIMEOUT"],
178             "A test can not be both SLOW and TIMEOUT. "
179             "If it times out indefinitely, then it should be just TIMEOUT. "
180             "passes/text.html  [test/expectations] [5]")
181         self.assert_lines_lint(
182             ["BUGWK1 : does/not/exist.html = FAIL"],
183             "Path does not exist. does/not/exist.html  [test/expectations] [2]")
184
185     def test_parse_expectations(self):
186         self.assert_lines_lint(
187             ["BUGWK1 : passes/text.html = PASS"],
188             "")
189         self.assert_lines_lint(
190             ["BUGWK1 : passes/text.html = UNSUPPORTED"],
191             "Unsupported expectation: unsupported "
192             "passes/text.html  [test/expectations] [5]")
193         self.assert_lines_lint(
194             ["BUGWK1 : passes/text.html = PASS UNSUPPORTED"],
195             "Unsupported expectation: unsupported "
196             "passes/text.html  [test/expectations] [5]")
197
198     def test_already_seen_test(self):
199         self.assert_lines_lint(
200             ["BUGWK1 : passes/text.html = PASS",
201              "BUGWK1 : passes/text.html = TIMEOUT"],
202             "Duplicate or ambiguous expectation. %s  [test/expectations] [5]" % self._test_file)
203
204         self.assert_lines_lint(
205             ["BUGWK1 LEOPARD : passes/text.html = PASS",
206              "BUGWK1 MAC : passes/text.html = TIMEOUT"],
207             "More specific entry on line 1 overrides line 2 passes/text.html  [test/expectations] [5]")
208
209         self.assert_lines_lint(
210             ["BUGWK1 LEOPARD : passes/text.html = PASS",
211              "BUGWK1 LEOPARD RELEASE : passes/text.html = TIMEOUT"],
212             "More specific entry on line 2 overrides line 1 passes/text.html  [test/expectations] [5]")
213
214         self.assert_lines_lint(
215             ["BUGWK1 RELEASE : passes/text.html = PASS",
216              "BUGWK1 CPU : passes/text.html = TIMEOUT"],
217             "Entries on line 1 and line 2 match overlapping sets of configurations passes/text.html  [test/expectations] [5]")
218
219         self.assert_lines_lint(
220             ["BUGWK1 WIN : passes/text.html = PASS",
221              "BUGWK1 MAC : passes/text.html = TIMEOUT"],
222             "")
223
224         self.assert_lines_lint(
225             ["BUGWK1 LEOPARD DEBUG : passes/text.html = PASS",
226              "BUGWK1 LEOPARD RELEASE : passes/text.html = TIMEOUT"],
227             "")
228
229
230     def test_tab(self):
231         self.assert_lines_lint(
232             ["\tBUGWK1 : passes/text.html = PASS"],
233             "Line contains tab character.  [whitespace/tab] [5]")