2010-08-26 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for test_expectations.py."""
31
32 import os
33 import sys
34 import unittest
35
36 from webkitpy.layout_tests import port
37 from webkitpy.layout_tests.layout_package.test_expectations import *
38
39 class FunctionsTest(unittest.TestCase):
40     def test_result_was_expected(self):
41         # test basics
42         self.assertEquals(result_was_expected(PASS, set([PASS]),
43                                               False, False), True)
44         self.assertEquals(result_was_expected(TEXT, set([PASS]),
45                                               False, False), False)
46
47         # test handling of FAIL expectations
48         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
49                                               False, False), True)
50         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
53                                               False, False), True)
54         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
55                                               False, False), False)
56
57         # test handling of SKIPped tests and results
58         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
59                                               False, True), True)
60         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
61                                               False, False), False)
62
63         # test handling of MISSING results and the REBASELINE modifier
64         self.assertEquals(result_was_expected(MISSING, set([PASS]),
65                                               True, False), True)
66         self.assertEquals(result_was_expected(MISSING, set([PASS]),
67                                               False, False), False)
68
69     def test_remove_pixel_failures(self):
70         self.assertEquals(remove_pixel_failures(set([TEXT])),
71                           set([TEXT]))
72         self.assertEquals(remove_pixel_failures(set([PASS])),
73                           set([PASS]))
74         self.assertEquals(remove_pixel_failures(set([IMAGE])),
75                           set([PASS]))
76         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
77                           set([TEXT]))
78         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
79                           set([PASS, CRASH]))
80
81
82 class Base(unittest.TestCase):
83     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
84         self._port = port.get('test', None)
85         self._exp = None
86         unittest.TestCase.__init__(self, testFunc)
87
88     def get_test(self, test_name):
89         return os.path.join(self._port.layout_tests_dir(), test_name)
90
91     def get_basic_tests(self):
92         return [self.get_test('failures/expected/text.html'),
93                 self.get_test('failures/expected/image_checksum.html'),
94                 self.get_test('failures/expected/crash.html'),
95                 self.get_test('failures/expected/missing_text.html'),
96                 self.get_test('failures/expected/image.html'),
97                 self.get_test('passes/text.html')]
98
99     def get_basic_expectations(self):
100         return """
101 BUG_TEST : failures/expected/text.html = TEXT
102 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
103 BUG_TEST REBASELINE : failure/expected/missing_image.html = MISSING
104 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
105 BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
106 """
107
108     def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
109                   is_debug_mode=False, tests_are_present=True):
110         self._exp = TestExpectations(self._port,
111              tests=self.get_basic_tests(),
112              expectations=expectations,
113              test_platform_name=self._port.test_platform_name(),
114              is_debug_mode=is_debug_mode,
115              is_lint_mode=is_lint_mode,
116              tests_are_present=tests_are_present,
117              overrides=overrides)
118
119     def assert_exp(self, test, result):
120         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
121                           set([result]))
122
123
124 class TestExpectationsTest(Base):
125     def test_basic(self):
126         self.parse_exp(self.get_basic_expectations())
127         self.assert_exp('failures/expected/text.html', TEXT)
128         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
129         self.assert_exp('passes/text.html', PASS)
130         self.assert_exp('failures/expected/image.html', PASS)
131
132     def test_multiple_results(self):
133         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
134         self.assertEqual(self._exp.get_expectations(
135             self.get_test('failures/expected/text.html')),
136             set([TEXT, CRASH]))
137
138     def test_defer(self):
139         self.parse_exp('BUGX DEFER : failures/expected/text.html = TEXT')
140         self.assertEqual(self._exp.get_options(
141             self.get_test('failures/expected/text.html')), ['bugx', 'defer'])
142
143     def test_precedence(self):
144         # This tests handling precedence of specific lines over directories
145         # and tests expectations covering entire directories.
146         exp_str = """
147 BUGX : failures/expected/text.html = TEXT
148 BUGX DEFER : failures/expected = IMAGE
149 """
150         self.parse_exp(exp_str)
151         self.assert_exp('failures/expected/text.html', TEXT)
152         self.assert_exp('failures/expected/crash.html', IMAGE)
153
154         self.parse_exp(exp_str, tests_are_present=False)
155         self.assert_exp('failures/expected/text.html', TEXT)
156         self.assert_exp('failures/expected/crash.html', IMAGE)
157
158     def test_release_mode(self):
159         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
160                        is_debug_mode=True)
161         self.assert_exp('failures/expected/text.html', TEXT)
162         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
163                        is_debug_mode=True)
164         self.assert_exp('failures/expected/text.html', PASS)
165         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
166                        is_debug_mode=False)
167         self.assert_exp('failures/expected/text.html', PASS)
168         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
169                        is_debug_mode=False)
170         self.assert_exp('failures/expected/text.html', TEXT)
171
172     def test_get_options(self):
173         self.parse_exp(self.get_basic_expectations())
174         self.assertEqual(self._exp.get_options(
175                          self.get_test('passes/text.html')), [])
176
177     def test_expectations_json_for_all_platforms(self):
178         self.parse_exp(self.get_basic_expectations())
179         json_str = self._exp.get_expectations_json_for_all_platforms()
180         # FIXME: test actual content?
181         self.assertTrue(json_str)
182
183     def test_get_expectations_string(self):
184         self.parse_exp(self.get_basic_expectations())
185         self.assertEquals(self._exp.get_expectations_string(
186                           self.get_test('failures/expected/text.html')),
187                           'TEXT')
188
189     def test_expectation_to_string(self):
190         # Normal cases are handled by other tests.
191         self.parse_exp(self.get_basic_expectations())
192         self.assertRaises(ValueError, self._exp.expectation_to_string,
193                           -1)
194
195     def test_get_test_set(self):
196         # Handle some corner cases for this routine not covered by other tests.
197         self.parse_exp(self.get_basic_expectations())
198         s = self._exp._expected_failures.get_test_set(WONTFIX)
199         self.assertEqual(s,
200             set([self.get_test('failures/expected/crash.html'),
201                  self.get_test('failures/expected/image_checksum.html')]))
202         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
203         self.assertEqual(s,
204             set([self.get_test('failures/expected/crash.html')]))
205         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
206                                                       include_skips=False)
207         self.assertEqual(s, set([]))
208
209     def test_syntax_missing_expectation(self):
210         # This is missing the expectation.
211         self.assertRaises(SyntaxError, self.parse_exp,
212                           'BUG_TEST: failures/expected/text.html',
213                           is_debug_mode=True)
214
215     def test_syntax_invalid_option(self):
216         self.assertRaises(SyntaxError, self.parse_exp,
217                           'BUG_TEST FOO: failures/expected/text.html = PASS')
218
219     def test_syntax_invalid_expectation(self):
220         # This is missing the expectation.
221         self.assertRaises(SyntaxError, self.parse_exp,
222                           'BUG_TEST: failures/expected/text.html = FOO')
223
224     def test_syntax_missing_bugid(self):
225         # This should log a non-fatal error.
226         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
227         self.assertEqual(
228             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
229
230     def test_semantic_slow_and_timeout(self):
231         # A test cannot be SLOW and expected to TIMEOUT.
232         self.assertRaises(SyntaxError, self.parse_exp,
233             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
234
235     def test_semantic_wontfix_defer(self):
236         # A test cannot be WONTFIX and DEFER.
237         self.assertRaises(SyntaxError, self.parse_exp,
238             'BUG_TEST WONTFIX DEFER : failures/expected/text.html = TEXT')
239
240     def test_semantic_rebaseline(self):
241         # Can't lint a file w/ 'REBASELINE' in it.
242         self.assertRaises(SyntaxError, self.parse_exp,
243             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
244             is_lint_mode=True)
245
246     def test_semantic_duplicates(self):
247         self.assertRaises(SyntaxError, self.parse_exp, """
248 BUG_TEST : failures/expected/text.html = TEXT
249 BUG_TEST : failures/expected/text.html = IMAGE""")
250
251         self.assertRaises(SyntaxError, self.parse_exp,
252             self.get_basic_expectations(), """
253 BUG_TEST : failures/expected/text.html = TEXT
254 BUG_TEST : failures/expected/text.html = IMAGE""")
255
256     def test_overrides(self):
257         self.parse_exp(self.get_basic_expectations(), """
258 BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
259         self.assert_exp('failures/expected/text.html', IMAGE)
260
261     def test_matches_an_expected_result(self):
262
263         def match(test, result, pixel_tests_enabled):
264             return self._exp.matches_an_expected_result(
265                 self.get_test(test), result, pixel_tests_enabled)
266
267         self.parse_exp(self.get_basic_expectations())
268         self.assertTrue(match('failures/expected/text.html', TEXT, True))
269         self.assertTrue(match('failures/expected/text.html', TEXT, False))
270         self.assertFalse(match('failures/expected/text.html', CRASH, True))
271         self.assertFalse(match('failures/expected/text.html', CRASH, False))
272         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
273                               True))
274         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
275                               False))
276         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
277         self.assertTrue(match('passes/text.html', PASS, False))
278
279
280 class RebaseliningTest(Base):
281     """Test rebaselining-specific functionality."""
282     def assertRemove(self, platform, input_expectations, expected_expectations):
283         self.parse_exp(input_expectations)
284         test = self.get_test('failures/expected/text.html')
285         actual_expectations = self._exp.remove_platform_from_expectations(
286             test, platform)
287         self.assertEqual(expected_expectations, actual_expectations)
288
289     def test_no_get_rebaselining_failures(self):
290         self.parse_exp(self.get_basic_expectations())
291         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
292
293     def test_get_rebaselining_failures_expand(self):
294         self.parse_exp("""
295 BUG_TEST REBASELINE : failures/expected/text.html = TEXT
296 """)
297         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
298
299     def test_remove_expand(self):
300         self.assertRemove('mac',
301             'BUGX REBASELINE : failures/expected/text.html = TEXT\n',
302             'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
303
304     def test_remove_mac_win(self):
305         self.assertRemove('mac',
306             'BUGX REBASELINE MAC WIN : failures/expected/text.html = TEXT\n',
307             'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
308
309     def test_remove_mac_mac(self):
310         self.assertRemove('mac',
311             'BUGX REBASELINE MAC : failures/expected/text.html = TEXT\n',
312             '')
313
314     def test_remove_nothing(self):
315         self.assertRemove('mac',
316             '\n\n',
317             '\n\n')
318
319
320 if __name__ == '__main__':
321     unittest.main()