Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for test_expectations.py."""
31
32 import os
33 import sys
34 import unittest
35
36 from webkitpy.layout_tests import port
37 from webkitpy.layout_tests.layout_package.test_expectations import *
38
39 class FunctionsTest(unittest.TestCase):
40     def test_result_was_expected(self):
41         # test basics
42         self.assertEquals(result_was_expected(PASS, set([PASS]),
43                                               False, False), True)
44         self.assertEquals(result_was_expected(TEXT, set([PASS]),
45                                               False, False), False)
46
47         # test handling of FAIL expectations
48         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
49                                               False, False), True)
50         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
53                                               False, False), True)
54         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
55                                               False, False), False)
56
57         # test handling of SKIPped tests and results
58         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
59                                               False, True), True)
60         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
61                                               False, False), False)
62
63         # test handling of MISSING results and the REBASELINE modifier
64         self.assertEquals(result_was_expected(MISSING, set([PASS]),
65                                               True, False), True)
66         self.assertEquals(result_was_expected(MISSING, set([PASS]),
67                                               False, False), False)
68
69     def test_remove_pixel_failures(self):
70         self.assertEquals(remove_pixel_failures(set([TEXT])),
71                           set([TEXT]))
72         self.assertEquals(remove_pixel_failures(set([PASS])),
73                           set([PASS]))
74         self.assertEquals(remove_pixel_failures(set([IMAGE])),
75                           set([PASS]))
76         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
77                           set([TEXT]))
78         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
79                           set([PASS, CRASH]))
80
81
82 class Base(unittest.TestCase):
83     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
84         self._port = port.get('test', None)
85         self._exp = None
86         unittest.TestCase.__init__(self, testFunc)
87
88     def get_test(self, test_name):
89         return os.path.join(self._port.layout_tests_dir(), test_name)
90
91     def get_basic_tests(self):
92         return [self.get_test('failures/expected/text.html'),
93                 self.get_test('failures/expected/image_checksum.html'),
94                 self.get_test('failures/expected/crash.html'),
95                 self.get_test('failures/expected/missing_text.html'),
96                 self.get_test('failures/expected/image.html'),
97                 self.get_test('passes/text.html')]
98
99     def get_basic_expectations(self):
100         return """
101 BUG_TEST : failures/expected/text.html = TEXT
102 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
103 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
104 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
105 BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
106 """
107
108     def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
109                   is_debug_mode=False):
110         self._exp = TestExpectations(self._port,
111              tests=self.get_basic_tests(),
112              expectations=expectations,
113              test_platform_name=self._port.test_platform_name(),
114              is_debug_mode=is_debug_mode,
115              is_lint_mode=is_lint_mode,
116              overrides=overrides)
117
118     def assert_exp(self, test, result):
119         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
120                           set([result]))
121
122
123 class TestExpectationsTest(Base):
124     def test_basic(self):
125         self.parse_exp(self.get_basic_expectations())
126         self.assert_exp('failures/expected/text.html', TEXT)
127         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
128         self.assert_exp('passes/text.html', PASS)
129         self.assert_exp('failures/expected/image.html', PASS)
130
131     def test_multiple_results(self):
132         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
133         self.assertEqual(self._exp.get_expectations(
134             self.get_test('failures/expected/text.html')),
135             set([TEXT, CRASH]))
136
137     def test_precedence(self):
138         # This tests handling precedence of specific lines over directories
139         # and tests expectations covering entire directories.
140         exp_str = """
141 BUGX : failures/expected/text.html = TEXT
142 BUGX WONTFIX : failures/expected = IMAGE
143 """
144         self.parse_exp(exp_str)
145         self.assert_exp('failures/expected/text.html', TEXT)
146         self.assert_exp('failures/expected/crash.html', IMAGE)
147
148     def test_category_expectations(self):
149         # This test checks unknown tests are not present in the
150         # expectations and that known test part of a test category is
151         # present in the expectations.
152         exp_str = """
153 BUGX WONTFIX : failures/expected = IMAGE
154 """
155         self.parse_exp(exp_str)
156         test_name = 'failures/expected/unknown-test.html'
157         unknown_test = self.get_test(test_name)
158         self.assertRaises(KeyError, self._exp.get_expectations,
159                           unknown_test)
160         self.assert_exp('failures/expected/crash.html', IMAGE)
161
162     def test_release_mode(self):
163         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
164                        is_debug_mode=True)
165         self.assert_exp('failures/expected/text.html', TEXT)
166         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
167                        is_debug_mode=True)
168         self.assert_exp('failures/expected/text.html', PASS)
169         self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
170                        is_debug_mode=False)
171         self.assert_exp('failures/expected/text.html', PASS)
172         self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
173                        is_debug_mode=False)
174         self.assert_exp('failures/expected/text.html', TEXT)
175
176     def test_get_options(self):
177         self.parse_exp(self.get_basic_expectations())
178         self.assertEqual(self._exp.get_options(
179                          self.get_test('passes/text.html')), [])
180
181     def test_expectations_json_for_all_platforms(self):
182         self.parse_exp(self.get_basic_expectations())
183         json_str = self._exp.get_expectations_json_for_all_platforms()
184         # FIXME: test actual content?
185         self.assertTrue(json_str)
186
187     def test_get_expectations_string(self):
188         self.parse_exp(self.get_basic_expectations())
189         self.assertEquals(self._exp.get_expectations_string(
190                           self.get_test('failures/expected/text.html')),
191                           'TEXT')
192
193     def test_expectation_to_string(self):
194         # Normal cases are handled by other tests.
195         self.parse_exp(self.get_basic_expectations())
196         self.assertRaises(ValueError, self._exp.expectation_to_string,
197                           -1)
198
199     def test_get_test_set(self):
200         # Handle some corner cases for this routine not covered by other tests.
201         self.parse_exp(self.get_basic_expectations())
202         s = self._exp._expected_failures.get_test_set(WONTFIX)
203         self.assertEqual(s,
204             set([self.get_test('failures/expected/crash.html'),
205                  self.get_test('failures/expected/image_checksum.html')]))
206         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
207         self.assertEqual(s,
208             set([self.get_test('failures/expected/crash.html')]))
209         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
210                                                       include_skips=False)
211         self.assertEqual(s, set([]))
212
213     def test_parse_error_fatal(self):
214         try:
215             self.parse_exp("""FOO : failures/expected/text.html = TEXT
216 SKIP : failures/expected/image.html""")
217             self.assertFalse(True, "ParseError wasn't raised")
218         except ParseError, e:
219             self.assertTrue(e.fatal)
220             exp_errors = [u'Line:1 Invalid modifier for test: foo failures/expected/text.html',
221                           u"Line:2 Missing expectations. [' failures/expected/image.html']"]
222             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
223             self.assertEqual(e.errors, exp_errors)
224
225     def test_parse_error_nonfatal(self):
226         try:
227             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
228                            is_lint_mode=True)
229             self.assertFalse(True, "ParseError wasn't raised")
230         except ParseError, e:
231             self.assertFalse(e.fatal)
232             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
233             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
234             self.assertEqual(e.errors, exp_errors)
235
236     def test_syntax_missing_expectation(self):
237         # This is missing the expectation.
238         self.assertRaises(ParseError, self.parse_exp,
239                           'BUG_TEST: failures/expected/text.html',
240                           is_debug_mode=True)
241
242     def test_syntax_invalid_option(self):
243         self.assertRaises(ParseError, self.parse_exp,
244                           'BUG_TEST FOO: failures/expected/text.html = PASS')
245
246     def test_syntax_invalid_expectation(self):
247         # This is missing the expectation.
248         self.assertRaises(ParseError, self.parse_exp,
249                           'BUG_TEST: failures/expected/text.html = FOO')
250
251     def test_syntax_missing_bugid(self):
252         # This should log a non-fatal error.
253         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
254         self.assertEqual(
255             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
256
257     def test_semantic_slow_and_timeout(self):
258         # A test cannot be SLOW and expected to TIMEOUT.
259         self.assertRaises(ParseError, self.parse_exp,
260             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
261
262     def test_semantic_rebaseline(self):
263         # Can't lint a file w/ 'REBASELINE' in it.
264         self.assertRaises(ParseError, self.parse_exp,
265             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
266             is_lint_mode=True)
267
268     def test_semantic_duplicates(self):
269         self.assertRaises(ParseError, self.parse_exp, """
270 BUG_TEST : failures/expected/text.html = TEXT
271 BUG_TEST : failures/expected/text.html = IMAGE""")
272
273         self.assertRaises(ParseError, self.parse_exp,
274             self.get_basic_expectations(), """
275 BUG_TEST : failures/expected/text.html = TEXT
276 BUG_TEST : failures/expected/text.html = IMAGE""")
277
278     def test_semantic_missing_file(self):
279         # This should log a non-fatal error.
280         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
281         self.assertEqual(
282             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
283
284
285     def test_overrides(self):
286         self.parse_exp(self.get_basic_expectations(), """
287 BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
288         self.assert_exp('failures/expected/text.html', IMAGE)
289
290     def test_matches_an_expected_result(self):
291
292         def match(test, result, pixel_tests_enabled):
293             return self._exp.matches_an_expected_result(
294                 self.get_test(test), result, pixel_tests_enabled)
295
296         self.parse_exp(self.get_basic_expectations())
297         self.assertTrue(match('failures/expected/text.html', TEXT, True))
298         self.assertTrue(match('failures/expected/text.html', TEXT, False))
299         self.assertFalse(match('failures/expected/text.html', CRASH, True))
300         self.assertFalse(match('failures/expected/text.html', CRASH, False))
301         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
302                               True))
303         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
304                               False))
305         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
306         self.assertTrue(match('passes/text.html', PASS, False))
307
308
309 class RebaseliningTest(Base):
310     """Test rebaselining-specific functionality."""
311     def assertRemove(self, platform, input_expectations, expected_expectations):
312         self.parse_exp(input_expectations)
313         test = self.get_test('failures/expected/text.html')
314         actual_expectations = self._exp.remove_platform_from_expectations(
315             test, platform)
316         self.assertEqual(expected_expectations, actual_expectations)
317
318     def test_no_get_rebaselining_failures(self):
319         self.parse_exp(self.get_basic_expectations())
320         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
321
322     def test_get_rebaselining_failures_expand(self):
323         self.parse_exp("""
324 BUG_TEST REBASELINE : failures/expected/text.html = TEXT
325 """)
326         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
327
328     def test_remove_expand(self):
329         self.assertRemove('mac',
330             'BUGX REBASELINE : failures/expected/text.html = TEXT\n',
331             'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
332
333     def test_remove_mac_win(self):
334         self.assertRemove('mac',
335             'BUGX REBASELINE MAC WIN : failures/expected/text.html = TEXT\n',
336             'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
337
338     def test_remove_mac_mac(self):
339         self.assertRemove('mac',
340             'BUGX REBASELINE MAC : failures/expected/text.html = TEXT\n',
341             '')
342
343     def test_remove_nothing(self):
344         self.assertRemove('mac',
345             '\n\n',
346             '\n\n')
347
348
349 if __name__ == '__main__':
350     unittest.main()