22214b054bc8bfc48041dc1333457f9dbf4e9dfb
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for test_expectations.py."""
31
32 import os
33 import sys
34 import unittest
35
36 from webkitpy.layout_tests import port
37 from webkitpy.layout_tests.layout_package.test_expectations import *
38
39 class FunctionsTest(unittest.TestCase):
40     def test_result_was_expected(self):
41         # test basics
42         self.assertEquals(result_was_expected(PASS, set([PASS]),
43                                               False, False), True)
44         self.assertEquals(result_was_expected(TEXT, set([PASS]),
45                                               False, False), False)
46
47         # test handling of FAIL expectations
48         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
49                                               False, False), True)
50         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
53                                               False, False), True)
54         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
55                                               False, False), False)
56
57         # test handling of SKIPped tests and results
58         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
59                                               False, True), True)
60         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
61                                               False, False), False)
62
63         # test handling of MISSING results and the REBASELINE modifier
64         self.assertEquals(result_was_expected(MISSING, set([PASS]),
65                                               True, False), True)
66         self.assertEquals(result_was_expected(MISSING, set([PASS]),
67                                               False, False), False)
68
69     def test_remove_pixel_failures(self):
70         self.assertEquals(remove_pixel_failures(set([TEXT])),
71                           set([TEXT]))
72         self.assertEquals(remove_pixel_failures(set([PASS])),
73                           set([PASS]))
74         self.assertEquals(remove_pixel_failures(set([IMAGE])),
75                           set([PASS]))
76         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
77                           set([TEXT]))
78         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
79                           set([PASS, CRASH]))
80
81
82 class Base(unittest.TestCase):
83     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
84         self._port = port.get('test', None)
85         self._exp = None
86         unittest.TestCase.__init__(self, testFunc)
87
88     def get_test(self, test_name):
89         return os.path.join(self._port.layout_tests_dir(), test_name)
90
91     def get_basic_tests(self):
92         return [self.get_test('failures/expected/text.html'),
93                 self.get_test('failures/expected/image_checksum.html'),
94                 self.get_test('failures/expected/crash.html'),
95                 self.get_test('failures/expected/missing_text.html'),
96                 self.get_test('passes/text.html')]
97
98     def get_basic_expectations(self):
99         return """
100 BUG_TEST : failures/expected/text.html = TEXT
101 BUG_TEST SKIP : failures/expected/crash.html = CRASH
102 BUG_TEST REBASELINE : failure/expected/missing_image.html = MISSING
103 BUG_TEST : failures/expected/image_checksum.html = IMAGE
104 """
105
106     def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
107                   is_debug_mode=False, tests_are_present=True):
108         self._exp = TestExpectations(self._port,
109              tests=self.get_basic_tests(),
110              expectations=expectations,
111              test_platform_name=self._port.test_platform_name(),
112              is_debug_mode=is_debug_mode,
113              is_lint_mode=is_lint_mode,
114              tests_are_present=tests_are_present,
115              overrides=overrides)
116
117     def assert_exp(self, test, result):
118         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
119                           set([result]))
120
121
122 class TestExpectationsTest(Base):
123     def test_basic(self):
124         self.parse_exp(self.get_basic_expectations())
125         self.assert_exp('failures/expected/text.html', TEXT)
126         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
127         self.assert_exp('passes/text.html', PASS)
128
129     def test_defer(self):
130         self.parse_exp('BUGX DEFER : failures/expected/text.html = TEXT')
131         self.assertEqual(self._exp.get_options(
132             self.get_test('failures/expected/text.html')), ['bugx', 'defer'])
133
134     def test_precedence(self):
135         # This tests handling precedence of specific lines over directories
136         # and tests expectations covering entire directories.
137         exp_str = """
138 BUGX : failures/expected/text.html = TEXT
139 BUGX DEFER : failures/expected = IMAGE
140 """
141         self.parse_exp(exp_str)
142         self.assert_exp('failures/expected/text.html', TEXT)
143         self.assert_exp('failures/expected/crash.html', IMAGE)
144
145         self.parse_exp(exp_str, tests_are_present=False)
146         self.assert_exp('failures/expected/text.html', TEXT)
147         self.assert_exp('failures/expected/crash.html', IMAGE)
148
149     def test_release_mode(self):
150         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
151                        is_debug_mode=True)
152         self.assert_exp('failures/expected/text.html', TEXT)
153         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
154                        is_debug_mode=True)
155         self.assert_exp('failures/expected/text.html', PASS)
156         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
157                        is_debug_mode=False)
158         self.assert_exp('failures/expected/text.html', PASS)
159         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
160                        is_debug_mode=False)
161         self.assert_exp('failures/expected/text.html', TEXT)
162
163     def test_get_options(self):
164         self.parse_exp(self.get_basic_expectations())
165         self.assertEqual(self._exp.get_options(
166                          self.get_test('passes/text.html')), [])
167
168     def test_expectations_json_for_all_platforms(self):
169         self.parse_exp(self.get_basic_expectations())
170         json_str = self._exp.get_expectations_json_for_all_platforms()
171         # FIXME: test actual content?
172         self.assertTrue(json_str)
173
174     def test_get_expectations_string(self):
175         self.parse_exp(self.get_basic_expectations())
176         self.assertEquals(self._exp.get_expectations_string(
177                           self.get_test('failures/expected/text.html')),
178                           'TEXT')
179
180     def test_expectation_to_string(self):
181         # Normal cases are handled by other tests.
182         self.parse_exp(self.get_basic_expectations())
183         self.assertRaises(ValueError, self._exp.expectation_to_string,
184                           -1)
185
186     def test_syntax_missing_expectation(self):
187         # This is missing the expectation.
188         self.assertRaises(SyntaxError, self.parse_exp,
189                           'BUG_TEST: failures/expected/text.html',
190                           is_debug_mode=True)
191
192     def test_syntax_invalid_option(self):
193         self.assertRaises(SyntaxError, self.parse_exp,
194                           'BUG_TEST FOO: failures/expected/text.html = PASS')
195
196     def test_syntax_invalid_expectation(self):
197         # This is missing the expectation.
198         self.assertRaises(SyntaxError, self.parse_exp,
199                           'BUG_TEST: failures/expected/text.html = FOO')
200
201     def test_syntax_missing_bugid(self):
202         # This should log a non-fatal error.
203         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
204         self.assertEqual(
205             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
206
207     def test_semantic_slow_and_timeout(self):
208         # A test cannot be SLOW and expected to TIMEOUT.
209         self.assertRaises(SyntaxError, self.parse_exp,
210             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
211
212     def test_semantic_wontfix_defer(self):
213         # A test cannot be WONTFIX and DEFER.
214         self.assertRaises(SyntaxError, self.parse_exp,
215             'BUG_TEST WONTFIX DEFER : failures/expected/text.html = TEXT')
216
217     def test_semantic_rebaseline(self):
218         # Can't lint a file w/ 'REBASELINE' in it.
219         self.assertRaises(SyntaxError, self.parse_exp,
220             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
221             is_lint_mode=True)
222
223     def test_semantic_duplicates(self):
224         self.assertRaises(SyntaxError, self.parse_exp, """
225 BUG_TEST : failures/expected/text.html = TEXT
226 BUG_TEST : failures/expected/text.html = IMAGE""")
227
228         self.assertRaises(SyntaxError, self.parse_exp,
229             self.get_basic_expectations(), """
230 BUG_TEST : failures/expected/text.html = TEXT
231 BUG_TEST : failures/expected/text.html = IMAGE""")
232
233     def test_overrides(self):
234         self.parse_exp(self.get_basic_expectations(), """
235 BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
236         self.assert_exp('failures/expected/text.html', IMAGE)
237
238     def test_matches_an_expected_result(self):
239
240         def match(test, result, pixel_tests_enabled):
241             return self._exp.matches_an_expected_result(
242                 self.get_test(test), result, pixel_tests_enabled)
243
244         self.parse_exp(self.get_basic_expectations())
245         self.assertTrue(match('failures/expected/text.html', TEXT, True))
246         self.assertTrue(match('failures/expected/text.html', TEXT, False))
247         self.assertFalse(match('failures/expected/text.html', CRASH, True))
248         self.assertFalse(match('failures/expected/text.html', CRASH, False))
249         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
250                               True))
251         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
252                               False))
253         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
254         self.assertTrue(match('passes/text.html', PASS, False))
255
256
257 class RebaseliningTest(Base):
258     """Test rebaselining-specific functionality."""
259     def test_no_get_rebaselining_failures(self):
260         self.parse_exp(self.get_basic_expectations())
261         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
262
263     def test_basic(self):
264         self.parse_exp("""
265 BUG_TEST REBASELINE : failures/expected/text.html = TEXT
266 """)
267         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
268
269         new_exp_str = self._exp.remove_platform_from_expectations(
270             self.get_test('failures/expected/text.html'), 'TEST')
271         # FIXME: actually test rebaselining
272         # self.assertEqual(new_exp_str, '\n')
273
274 if __name__ == '__main__':
275     unittest.main()