4d3f0b6676ba30336d2f33dd2c310d8987e4dcee
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2013 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from collections import OrderedDict
33
34 from webkitpy.common.host_mock import MockHost
35 from webkitpy.common.system.outputcapture import OutputCapture
36
37 from webkitpy.layout_tests.models.test_configuration import *
38 from webkitpy.layout_tests.models.test_expectations import *
39 from webkitpy.layout_tests.models.test_configuration import *
40
41
42 class Base(unittest.TestCase):
43     # Note that all of these tests are written assuming the configuration
44     # being tested is Windows XP, Release build.
45
46     def __init__(self, testFunc):
47         host = MockHost()
48         self._port = host.port_factory.get('test-win-xp', None)
49         self._exp = None
50         unittest.TestCase.__init__(self, testFunc)
51
52     def get_basic_tests(self):
53         return ['failures/expected/text.html',
54                 'failures/expected/image_checksum.html',
55                 'failures/expected/crash.html',
56                 'failures/expected/leak.html',
57                 'failures/expected/flaky-leak.html',
58                 'failures/expected/missing_text.html',
59                 'failures/expected/image.html',
60                 'failures/expected/reftest.html',
61                 'failures/expected/leaky-reftest.html',
62                 'passes/text.html']
63
64     def get_basic_expectations(self):
65         return """
66 Bug(test) failures/expected/text.html [ Failure ]
67 Bug(test) failures/expected/crash.html [ WontFix ]
68 Bug(test) failures/expected/leak.html [ Leak ]
69 Bug(test) failures/expected/flaky-leak.html [ Failure Leak ]
70 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
71 Bug(test) failures/expected/image_checksum.html [ WontFix ]
72 Bug(test) failures/expected/image.html [ WontFix Mac ]
73 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
74 Bug(test) failures/expected/leaky-reftest.html [ ImageOnlyFailure Leak ]
75 """
76
77     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
78         expectations_dict = OrderedDict()
79         expectations_dict['expectations'] = expectations
80         if overrides:
81             expectations_dict['overrides'] = overrides
82         self._port.expectations_dict = lambda: expectations_dict
83         expectations_to_lint = expectations_dict if is_lint_mode else None
84         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
85         self._exp.parse_all_expectations()
86
87     def assert_exp(self, test, result):
88         self.assertEqual(self._exp.model().get_expectations(test), set([result]))
89
90     def assert_exp_set(self, test, result_set):
91         self.assertEqual(self._exp.model().get_expectations(test), result_set)
92
93     def assert_bad_expectations(self, expectations, overrides=None):
94         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
95
96
97 class BasicTests(Base):
98     def test_basic(self):
99         self.parse_exp(self.get_basic_expectations())
100         self.assert_exp('failures/expected/text.html', FAIL)
101         self.assert_exp('failures/expected/image_checksum.html', PASS)
102         self.assert_exp('failures/expected/reftest.html', IMAGE)
103         self.assert_exp_set('failures/expected/leaky-reftest.html', set([IMAGE, LEAK]))
104         self.assert_exp('failures/expected/leak.html', LEAK)
105         self.assert_exp_set('failures/expected/flaky-leak.html', set([FAIL, LEAK]))
106         self.assert_exp('passes/text.html', PASS)
107         # self.assert_exp_set('passes/flaky-leak.html', set([PASS, LEAK]))
108         self.assert_exp('failures/expected/image.html', PASS)
109
110
111 class MiscTests(Base):
112     def test_multiple_results(self):
113         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
114         self.assertEqual(self._exp.model().get_expectations(
115             'failures/expected/text.html'),
116             set([FAIL, CRASH]))
117
118     def test_result_was_expected(self):
119         # test basics
120         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
121         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
122
123         # test handling of SKIPped tests and results
124         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
125         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
126
127         # test handling of MISSING results and the REBASELINE modifier
128         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
129         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
130
131     def test_remove_pixel_failures(self):
132         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
133         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
134         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
135         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
136         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
137
138     def test_remove_leak_failures(self):
139         self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL])), set([FAIL]))
140         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS])), set([PASS]))
141         self.assertEqual(TestExpectations.remove_leak_failures(set([LEAK])), set([PASS]))
142         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, LEAK])), set([PASS]))
143         self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL, LEAK])), set([FAIL]))
144         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, IMAGE, LEAK, CRASH])), set([PASS, IMAGE, CRASH]))
145
146     def test_suffixes_for_expectations(self):
147         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
148         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
149         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
150         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
151
152     def test_category_expectations(self):
153         # This test checks unknown tests are not present in the
154         # expectations and that known test part of a test category is
155         # present in the expectations.
156         exp_str = 'Bug(x) failures/expected [ WontFix ]'
157         self.parse_exp(exp_str)
158         test_name = 'failures/expected/unknown-test.html'
159         unknown_test = test_name
160         self.assertRaises(KeyError, self._exp.model().get_expectations,
161                           unknown_test)
162         self.assert_exp('failures/expected/crash.html', PASS)
163
164     def test_get_modifiers(self):
165         self.parse_exp(self.get_basic_expectations())
166         self.assertEqual(self._exp.model().get_modifiers('passes/text.html'), [])
167
168     def test_get_expectations_string(self):
169         self.parse_exp(self.get_basic_expectations())
170         self.assertEqual(self._exp.model().get_expectations_string('failures/expected/text.html'), 'FAIL')
171
172     def test_expectation_to_string(self):
173         # Normal cases are handled by other tests.
174         self.parse_exp(self.get_basic_expectations())
175         self.assertRaises(ValueError, self._exp.model().expectation_to_string, -1)
176
177     def test_get_test_set(self):
178         # Handle some corner cases for this routine not covered by other tests.
179         self.parse_exp(self.get_basic_expectations())
180         s = self._exp.model().get_test_set(WONTFIX)
181         self.assertEqual(s,
182             set(['failures/expected/crash.html',
183                  'failures/expected/image_checksum.html']))
184
185     def test_parse_warning(self):
186         try:
187             filesystem = self._port.host.filesystem
188             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
189             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
190                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
191                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
192             self.assertFalse(True, "ParseError wasn't raised")
193         except ParseError as e:
194             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
195                         "expectations:2 Path does not exist. non-existent-test.html")
196             self.assertEqual(str(e), warnings)
197
198     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
199         oc = OutputCapture()
200         try:
201             oc.capture_output()
202             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
203         finally:
204             _, _, logs = oc.restore_output()
205             self.assertNotEquals(logs, '')
206
207     def test_error_on_different_platform(self):
208         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
209         self.assertRaises(ParseError, self.parse_exp,
210             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
211             is_lint_mode=True)
212
213     def test_error_on_different_build_type(self):
214         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
215         self.assertRaises(ParseError, self.parse_exp,
216             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
217             is_lint_mode=True)
218
219     def test_overrides(self):
220         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
221                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
222         self.assert_exp('failures/expected/text.html', IMAGE)
223
224     def test_overrides__directory(self):
225         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
226                        "Bug(override) failures/expected [ Crash ]")
227         self.assert_exp('failures/expected/text.html', CRASH)
228         self.assert_exp('failures/expected/image.html', CRASH)
229
230     def test_overrides__duplicate(self):
231         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
232                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
233                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
234
235     def test_pixel_tests_flag(self):
236         def match(test, result, pixel_tests_enabled):
237             return self._exp.matches_an_expected_result(test, result, pixel_tests_enabled, False)
238
239         self.parse_exp(self.get_basic_expectations())
240         pixel_tests_enabled = True
241         pixel_tests_disabled = False
242         self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_enabled))
243         self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_disabled))
244         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled))
245         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_disabled))
246         self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_enabled))
247         self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_disabled))
248         self.assertTrue(match('failures/expected/crash.html', PASS, pixel_tests_disabled))
249         self.assertTrue(match('passes/text.html', PASS, pixel_tests_disabled))
250
251     def test_world_leaks_flag(self):
252         def match(test, result, pixel_tests_enabled, world_leaks_enabled):
253             return self._exp.matches_an_expected_result(test, result, pixel_tests_enabled, world_leaks_enabled)
254
255         pixel_tests_enabled = True
256         pixel_tests_disabled = False
257         world_leaks_enabled = True
258         world_leaks_disabled = False
259
260         self.parse_exp(self.get_basic_expectations())
261         self.assertTrue(match('failures/expected/leak.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
262         self.assertTrue(match('failures/expected/leak.html', PASS, pixel_tests_enabled, world_leaks_disabled))
263         self.assertTrue(match('failures/expected/flaky-leak.html', FAIL, pixel_tests_enabled, world_leaks_disabled))
264
265         self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_disabled, world_leaks_enabled))
266         self.assertTrue(match('failures/expected/leaky-reftest.html', PASS, pixel_tests_disabled, world_leaks_disabled))
267
268         self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_enabled))
269         self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
270         self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_disabled))
271
272         self.assertFalse(match('failures/expected/text.html', PASS, pixel_tests_enabled, world_leaks_enabled))
273         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled, world_leaks_disabled))
274         self.assertTrue(match('passes/text.html', PASS, pixel_tests_enabled, world_leaks_disabled))
275
276     def test_more_specific_override_resets_skip(self):
277         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
278                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
279         self.assert_exp('failures/expected/text.html', IMAGE)
280         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
281                                                      'failures/expected/text.html') in
282                          self._exp.model().get_tests_with_result_type(SKIP))
283
284
285 class SkippedTests(Base):
286     def check(self, expectations, overrides, skips, lint=False):
287         port = MockHost().port_factory.get('mac')
288         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
289         expectations_dict = OrderedDict()
290         expectations_dict['expectations'] = expectations
291         if overrides:
292             expectations_dict['overrides'] = overrides
293         port.expectations_dict = lambda: expectations_dict
294         port.skipped_layout_tests = lambda tests: set(skips)
295         expectations_to_lint = expectations_dict if lint else None
296         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
297         exp.parse_all_expectations()
298
299         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
300         self.assertEqual(exp.model().get_modifiers('failures/expected/text.html'),
301                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
302         self.assertEqual(exp.model().get_expectations('failures/expected/text.html'), set([PASS]))
303
304     def test_skipped_tests_work(self):
305         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
306
307     def test_duplicate_skipped_test_fails_lint(self):
308         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
309
310     def test_skipped_file_overrides_expectations(self):
311         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
312                    skips=['failures/expected/text.html'])
313
314     def test_skipped_dir_overrides_expectations(self):
315         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
316                    skips=['failures/expected'])
317
318     def test_skipped_file_overrides_overrides(self):
319         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
320                    skips=['failures/expected/text.html'])
321
322     def test_skipped_dir_overrides_overrides(self):
323         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
324                    skips=['failures/expected'])
325
326     def test_skipped_entry_dont_exist(self):
327         port = MockHost().port_factory.get('mac')
328         expectations_dict = OrderedDict()
329         expectations_dict['expectations'] = ''
330         port.expectations_dict = lambda: expectations_dict
331         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
332         capture = OutputCapture()
333         capture.capture_output()
334         exp = TestExpectations(port)
335         exp.parse_all_expectations()
336         _, _, logs = capture.restore_output()
337         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
338
339
340 class ExpectationSyntaxTests(Base):
341     def test_unrecognized_expectation(self):
342         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
343
344     def test_macro(self):
345         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
346         self.parse_exp(exp_str)
347         self.assert_exp('failures/expected/text.html', FAIL)
348
349     def assert_tokenize_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
350         bugs = bugs or []
351         modifiers = modifiers or []
352         expectations = expectations or []
353         warnings = warnings or []
354         filename = 'TestExpectations'
355         line_number = 1
356         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
357         self.assertEqual(expectation_line.warnings, warnings)
358         self.assertEqual(expectation_line.name, name)
359         self.assertEqual(expectation_line.filename, filename)
360         self.assertEqual(expectation_line.line_number, line_number)
361         if not warnings:
362             self.assertEqual(expectation_line.modifiers, modifiers)
363             self.assertEqual(expectation_line.expectations, expectations)
364
365     def test_bare_name(self):
366         self.assert_tokenize_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
367
368     def test_bare_name_and_bugs(self):
369         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
370         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
371         self.assert_tokenize_exp('webkit.org/b/12345 webkit.org/b/34567 foo.html', modifiers=['BUGWK12345', 'BUGWK34567', 'SKIP'], expectations=['PASS'])
372
373     def test_comments(self):
374         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
375         self.assert_tokenize_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
376
377     def test_config_modifiers(self):
378         self.assert_tokenize_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
379         self.assert_tokenize_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
380         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
381
382     def test_unknown_config(self):
383         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
384
385     def test_unknown_expectation(self):
386         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
387
388     def test_skip(self):
389         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
390
391     def test_slow(self):
392         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
393
394     def test_leak(self):
395         self.assert_tokenize_exp('foo.html [ Leak ]', modifiers=[], expectations=['LEAK'])
396
397     def test_wontfix(self):
398         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
399         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
400         self.assert_tokenize_exp('foo.html [ WontFix Pass Failure ]', modifiers=['WONTFIX'], expectations=['PASS', 'FAIL'])
401
402     def test_blank_line(self):
403         self.assert_tokenize_exp('', name=None)
404
405     def test_warnings(self):
406         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
407         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
408         self.assert_tokenize_exp('webkit.org/b/12345 ]', warnings=['unexpected "]"'], name=None)
409
410         self.assert_tokenize_exp('foo.html webkit.org/b/12345 ]', warnings=['"webkit.org/b/12345" is not at the start of the line.'])
411
412
413 class SemanticTests(Base):
414     def test_bug_format(self):
415         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
416
417     def test_bad_bugid(self):
418         try:
419             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
420             self.fail('should have raised an error about a bad bug identifier')
421         except ParseError as exp:
422             self.assertEqual(len(exp.warnings), 1)
423
424     def test_missing_bugid(self):
425         self.parse_exp('failures/expected/text.html [ Failure ]')
426         self.assertFalse(self._exp.has_warnings())
427
428         self._port.warn_if_bug_missing_in_test_expectations = lambda: True
429
430         self.parse_exp('failures/expected/text.html [ Failure ]')
431         line = self._exp._model.get_expectation_line('failures/expected/text.html')
432         self.assertFalse(line.is_invalid())
433         self.assertEqual(line.warnings, ['Test lacks BUG modifier.'])
434
435     def test_skip_and_wontfix(self):
436         # Skip is not allowed to have other expectations as well, because those
437         # expectations won't be exercised and may become stale .
438         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
439         self.assertTrue(self._exp.has_warnings())
440
441         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
442         self.assertFalse(self._exp.has_warnings())
443
444         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
445         self.assertFalse(self._exp.has_warnings())
446
447     def test_slow_and_timeout(self):
448         # A test cannot be SLOW and expected to TIMEOUT.
449         self.assertRaises(ParseError, self.parse_exp,
450             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
451
452     def test_rebaseline(self):
453         # Can't lint a file w/ 'REBASELINE' in it.
454         self.assertRaises(ParseError, self.parse_exp,
455             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
456             is_lint_mode=True)
457
458     def test_duplicates(self):
459         self.assertRaises(ParseError, self.parse_exp, """
460 Bug(exp) failures/expected/text.html [ Failure ]
461 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
462
463         self.assertRaises(ParseError, self.parse_exp,
464             self.get_basic_expectations(), overrides="""
465 Bug(override) failures/expected/text.html [ Failure ]
466 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
467
468     def test_missing_file(self):
469         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
470         self.assertTrue(self._exp.has_warnings(), 1)
471
472
473 class PrecedenceTests(Base):
474     def test_file_over_directory(self):
475         # This tests handling precedence of specific lines over directories
476         # and tests expectations covering entire directories.
477         exp_str = """
478 Bug(x) failures/expected/text.html [ Failure ]
479 Bug(y) failures/expected [ WontFix ]
480 """
481         self.parse_exp(exp_str)
482         self.assert_exp('failures/expected/text.html', FAIL)
483         self.assert_exp('failures/expected/crash.html', PASS)
484
485         exp_str = """
486 Bug(x) failures/expected [ WontFix ]
487 Bug(y) failures/expected/text.html [ Failure ]
488 """
489         self.parse_exp(exp_str)
490         self.assert_exp('failures/expected/text.html', FAIL)
491         self.assert_exp('failures/expected/crash.html', PASS)
492
493     def test_ambiguous(self):
494         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
495                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
496
497     def test_more_modifiers(self):
498         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
499                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
500
501     def test_order_in_file(self):
502         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
503                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
504
505     def test_macro_overrides(self):
506         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
507                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
508
509
510 class RemoveConfigurationsTest(Base):
511     def test_remove(self):
512         host = MockHost()
513         test_port = host.port_factory.get('test-win-xp', None)
514         test_port.test_exists = lambda test: True
515         test_port.test_isfile = lambda test: True
516
517         test_config = test_port.test_configuration()
518         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
519 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
520 """}
521         expectations = TestExpectations(test_port, self.get_basic_tests())
522         expectations.parse_all_expectations()
523
524         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
525
526         self.assertEqual("""Bug(x) [ 7SP0 Linux Vista Release ] failures/expected/foo.html [ Failure ]
527 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
528 """, actual_expectations)
529
530     def test_remove_line(self):
531         host = MockHost()
532         test_port = host.port_factory.get('test-win-xp', None)
533         test_port.test_exists = lambda test: True
534         test_port.test_isfile = lambda test: True
535
536         test_config = test_port.test_configuration()
537         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
538 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
539 """}
540         expectations = TestExpectations(test_port)
541         expectations.parse_all_expectations()
542
543         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
544         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
545         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-7sp0', None).test_configuration())
546
547         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
548 """, actual_expectations)
549
550
551 class RebaseliningTest(Base):
552     """Test rebaselining-specific functionality."""
553     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
554         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
555         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
556         self.assertEqual(expected_expectations, actual_expectations)
557         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
558         self.assertEqual(expected_overrides, actual_overrides)
559
560     def test_remove(self):
561         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
562                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
563                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
564                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
565                           ['failures/expected/text.html'],
566                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
567                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
568                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
569
570         # Ensure that we don't modify unrelated lines, even if we could rewrite them.
571         # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html"
572         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
573                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
574                           'Bug(z) failures/expected/crash.html\n',
575                           '',
576                           ['failures/expected/text.html'],
577                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
578                           'Bug(z) failures/expected/crash.html\n',
579                           '')
580
581     def test_get_rebaselining_failures(self):
582         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
583         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
584         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
585
586         self.parse_exp(self.get_basic_expectations())
587         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
588
589
590 class TestExpectationSerializationTests(unittest.TestCase):
591     def __init__(self, testFunc):
592         host = MockHost()
593         test_port = host.port_factory.get('test-win-xp', None)
594         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
595         unittest.TestCase.__init__(self, testFunc)
596
597     def _tokenize(self, line):
598         return TestExpectationParser._tokenize_line('path', line, 0)
599
600     def assert_round_trip(self, in_string, expected_string=None):
601         expectation = self._tokenize(in_string)
602         if expected_string is None:
603             expected_string = in_string
604         self.assertEqual(expected_string, expectation.to_string(self._converter))
605
606     def assert_list_round_trip(self, in_string, expected_string=None):
607         host = MockHost()
608         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
609         expectations = parser.parse('path', in_string)
610         if expected_string is None:
611             expected_string = in_string
612         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
613
614     def test_unparsed_to_string(self):
615         expectation = TestExpectationLine()
616
617         self.assertEqual(expectation.to_string(self._converter), '')
618         expectation.comment = ' Qux.'
619         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
620         expectation.name = 'bar'
621         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
622         expectation.modifiers = ['foo']
623         # FIXME: case should be preserved here but we can't until we drop the old syntax.
624         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
625         expectation.expectations = ['bAz']
626         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
627         expectation.expectations = ['bAz1', 'baZ2']
628         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
629         expectation.modifiers = ['foo1', 'foO2']
630         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
631         expectation.warnings.append('Oh the horror.')
632         self.assertEqual(expectation.to_string(self._converter), '')
633         expectation.original_string = 'Yes it is!'
634         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
635
636     def test_unparsed_list_to_string(self):
637         expectation = TestExpectationLine()
638         expectation.comment = 'Qux.'
639         expectation.name = 'bar'
640         expectation.modifiers = ['foo']
641         expectation.expectations = ['bAz1', 'baZ2']
642         # FIXME: case should be preserved here but we can't until we drop the old syntax.
643         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
644
645     def test_parsed_to_string(self):
646         expectation_line = TestExpectationLine()
647         expectation_line.parsed_bug_modifiers = ['BUGX']
648         expectation_line.name = 'test/name/for/realz.html'
649         expectation_line.parsed_expectations = set([IMAGE])
650         self.assertEqual(expectation_line.to_string(self._converter), None)
651         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
652         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
653         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
654         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
655
656     def test_serialize_parsed_expectations(self):
657         expectation_line = TestExpectationLine()
658         expectation_line.parsed_expectations = set([])
659         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
660         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
661         expectation_line.parsed_expectations = set([FAIL])
662         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
663         expectation_line.parsed_expectations = set([PASS, IMAGE])
664         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
665         expectation_line.parsed_expectations = set([FAIL, PASS])
666         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
667
668     def test_serialize_parsed_modifier_string(self):
669         expectation_line = TestExpectationLine()
670         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
671         expectation_line.parsed_modifiers = ['for', 'the']
672         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
673         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
674         expectation_line.parsed_bug_modifiers = []
675         expectation_line.parsed_modifiers = []
676         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
677         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
678         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
679         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
680         expectation_line.parsed_bug_modifiers = []
681         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
682         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
683
684     def test_format_line(self):
685         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
686         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
687
688     def test_string_roundtrip(self):
689         self.assert_round_trip('')
690         self.assert_round_trip('FOO')
691         self.assert_round_trip('[')
692         self.assert_round_trip('FOO [')
693         self.assert_round_trip('FOO ] bar')
694         self.assert_round_trip('  FOO [')
695         self.assert_round_trip('    [ FOO ] ')
696         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
697         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
698         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
699         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
700         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
701         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
702         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
703         self.assert_round_trip('FOO ] ] bar ==== BAZ')
704         self.assert_round_trip('=')
705         self.assert_round_trip('#')
706         self.assert_round_trip('# ')
707         self.assert_round_trip('# Foo')
708         self.assert_round_trip('# Foo')
709         self.assert_round_trip('# Foo :')
710         self.assert_round_trip('# Foo : =')
711
712     def test_list_roundtrip(self):
713         self.assert_list_round_trip('')
714         self.assert_list_round_trip('\n')
715         self.assert_list_round_trip('\n\n')
716         self.assert_list_round_trip('bar')
717         self.assert_list_round_trip('bar\n# Qux.')
718         self.assert_list_round_trip('bar\n# Qux.\n')
719
720     def test_reconstitute_only_these(self):
721         lines = []
722         reconstitute_only_these = []
723
724         def add_line(matching_configurations, reconstitute):
725             expectation_line = TestExpectationLine()
726             expectation_line.original_string = "Nay"
727             expectation_line.parsed_bug_modifiers = ['BUGX']
728             expectation_line.name = 'Yay'
729             expectation_line.parsed_expectations = set([IMAGE])
730             expectation_line.matching_configurations = matching_configurations
731             lines.append(expectation_line)
732             if reconstitute:
733                 reconstitute_only_these.append(expectation_line)
734
735         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
736         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
737         serialized = TestExpectations.list_to_string(lines, self._converter)
738         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
739         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
740         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
741
742     def disabled_test_string_whitespace_stripping(self):
743         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
744         self.assert_round_trip('\n', '')
745         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
746         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
747         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
748         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
749         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
750         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')