webkitpy: make warning about missing BUG identifiers per-port configurable
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.outputcapture import OutputCapture
34
35 from webkitpy.layout_tests.models.test_configuration import *
36 from webkitpy.layout_tests.models.test_expectations import *
37 from webkitpy.layout_tests.models.test_configuration import *
38
39 try:
40     from collections import OrderedDict
41 except ImportError:
42     # Needed for Python < 2.7
43     from webkitpy.thirdparty.ordered_dict import OrderedDict
44
45
46 class Base(unittest.TestCase):
47     # Note that all of these tests are written assuming the configuration
48     # being tested is Windows XP, Release build.
49
50     def __init__(self, testFunc):
51         host = MockHost()
52         self._port = host.port_factory.get('test-win-xp', None)
53         self._exp = None
54         unittest.TestCase.__init__(self, testFunc)
55
56     def get_test(self, test_name):
57         # FIXME: Remove this routine and just reference test names directly.
58         return test_name
59
60     def get_basic_tests(self):
61         return [self.get_test('failures/expected/text.html'),
62                 self.get_test('failures/expected/image_checksum.html'),
63                 self.get_test('failures/expected/crash.html'),
64                 self.get_test('failures/expected/missing_text.html'),
65                 self.get_test('failures/expected/image.html'),
66                 self.get_test('passes/text.html')]
67
68     def get_basic_expectations(self):
69         return """
70 Bug(test) failures/expected/text.html [ Failure ]
71 Bug(test) failures/expected/crash.html [ WontFix ]
72 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
73 Bug(test) failures/expected/image_checksum.html [ WontFix ]
74 Bug(test) failures/expected/image.html [ WontFix Mac ]
75 """
76
77     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
78         expectations_dict = OrderedDict()
79         expectations_dict['expectations'] = expectations
80         if overrides:
81             expectations_dict['overrides'] = overrides
82         self._port.expectations_dict = lambda: expectations_dict
83         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
84
85     def assert_exp(self, test, result):
86         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
87                           set([result]))
88
89     def assert_bad_expectations(self, expectations, overrides=None):
90         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
91
92
93 class BasicTests(Base):
94     def test_basic(self):
95         self.parse_exp(self.get_basic_expectations())
96         self.assert_exp('failures/expected/text.html', FAIL)
97         self.assert_exp('failures/expected/image_checksum.html', PASS)
98         self.assert_exp('passes/text.html', PASS)
99         self.assert_exp('failures/expected/image.html', PASS)
100
101
102 class MiscTests(Base):
103     def test_multiple_results(self):
104         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
105         self.assertEqual(self._exp.get_expectations(
106             self.get_test('failures/expected/text.html')),
107             set([FAIL, CRASH]))
108
109     def test_result_was_expected(self):
110         # test basics
111         self.assertEquals(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
112         self.assertEquals(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
113
114         # test handling of SKIPped tests and results
115         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
116         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
117
118         # test handling of MISSING results and the REBASELINE modifier
119         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
120         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
121
122     def test_remove_pixel_failures(self):
123         self.assertEquals(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
124         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
125         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
126         self.assertEquals(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
127         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
128
129     def test_suffixes_for_expectations(self):
130         self.assertEquals(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
131         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
132         self.assertEquals(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
133         self.assertEquals(TestExpectations.suffixes_for_expectations(set()), set())
134
135     def test_category_expectations(self):
136         # This test checks unknown tests are not present in the
137         # expectations and that known test part of a test category is
138         # present in the expectations.
139         exp_str = 'Bug(x) failures/expected [ WontFix ]'
140         self.parse_exp(exp_str)
141         test_name = 'failures/expected/unknown-test.html'
142         unknown_test = self.get_test(test_name)
143         self.assertRaises(KeyError, self._exp.get_expectations,
144                           unknown_test)
145         self.assert_exp('failures/expected/crash.html', PASS)
146
147     def test_get_modifiers(self):
148         self.parse_exp(self.get_basic_expectations())
149         self.assertEqual(self._exp.get_modifiers(
150                          self.get_test('passes/text.html')), [])
151
152     def test_get_expectations_string(self):
153         self.parse_exp(self.get_basic_expectations())
154         self.assertEquals(self._exp.get_expectations_string(
155                           self.get_test('failures/expected/text.html')),
156                           'FAIL')
157
158     def test_expectation_to_string(self):
159         # Normal cases are handled by other tests.
160         self.parse_exp(self.get_basic_expectations())
161         self.assertRaises(ValueError, self._exp.expectation_to_string,
162                           -1)
163
164     def test_get_test_set(self):
165         # Handle some corner cases for this routine not covered by other tests.
166         self.parse_exp(self.get_basic_expectations())
167         s = self._exp.get_test_set(WONTFIX)
168         self.assertEqual(s,
169             set([self.get_test('failures/expected/crash.html'),
170                  self.get_test('failures/expected/image_checksum.html')]))
171
172     def test_parse_warning(self):
173         try:
174             filesystem = self._port.host.filesystem
175             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
176             self.get_test('disabled-test.html-disabled'),
177             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
178                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
179                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
180             self.assertFalse(True, "ParseError wasn't raised")
181         except ParseError, e:
182             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
183                         "expectations:2 Path does not exist. non-existent-test.html")
184             self.assertEqual(str(e), warnings)
185
186     def test_error_on_different_platform(self):
187         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
188         self.assertRaises(ParseError, self.parse_exp,
189             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
190             is_lint_mode=True)
191
192     def test_error_on_different_build_type(self):
193         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
194         self.assertRaises(ParseError, self.parse_exp,
195             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
196             is_lint_mode=True)
197
198     def test_overrides(self):
199         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
200                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
201         self.assert_exp('failures/expected/text.html', IMAGE)
202
203     def test_overrides__directory(self):
204         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
205                        "Bug(override) failures/expected [ Crash ]")
206         self.assert_exp('failures/expected/text.html', CRASH)
207         self.assert_exp('failures/expected/image.html', CRASH)
208
209     def test_overrides__duplicate(self):
210         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
211                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
212                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
213
214     def test_pixel_tests_flag(self):
215         def match(test, result, pixel_tests_enabled):
216             return self._exp.matches_an_expected_result(
217                 self.get_test(test), result, pixel_tests_enabled)
218
219         self.parse_exp(self.get_basic_expectations())
220         self.assertTrue(match('failures/expected/text.html', FAIL, True))
221         self.assertTrue(match('failures/expected/text.html', FAIL, False))
222         self.assertFalse(match('failures/expected/text.html', CRASH, True))
223         self.assertFalse(match('failures/expected/text.html', CRASH, False))
224         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
225                               True))
226         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
227                               False))
228         self.assertTrue(match('failures/expected/crash.html', PASS, False))
229         self.assertTrue(match('passes/text.html', PASS, False))
230
231     def test_more_specific_override_resets_skip(self):
232         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
233                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
234         self.assert_exp('failures/expected/text.html', IMAGE)
235         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
236                                                      'failures/expected/text.html') in
237                          self._exp.get_tests_with_result_type(SKIP))
238
239
240 class SkippedTests(Base):
241     def check(self, expectations, overrides, skips, lint=False):
242         port = MockHost().port_factory.get('qt')
243         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
244         expectations_dict = OrderedDict()
245         expectations_dict['expectations'] = expectations
246         if overrides:
247             expectations_dict['overrides'] = overrides
248         port.expectations_dict = lambda: expectations_dict
249         port.skipped_layout_tests = lambda tests: set(skips)
250         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
251
252         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
253         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
254                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
255         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
256
257     def test_skipped_tests_work(self):
258         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
259
260     def test_duplicate_skipped_test_fails_lint(self):
261         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
262
263     def test_skipped_file_overrides_expectations(self):
264         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
265                    skips=['failures/expected/text.html'])
266
267     def test_skipped_dir_overrides_expectations(self):
268         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
269                    skips=['failures/expected'])
270
271     def test_skipped_file_overrides_overrides(self):
272         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
273                    skips=['failures/expected/text.html'])
274
275     def test_skipped_dir_overrides_overrides(self):
276         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
277                    skips=['failures/expected'])
278
279     def test_skipped_entry_dont_exist(self):
280         port = MockHost().port_factory.get('qt')
281         expectations_dict = OrderedDict()
282         expectations_dict['expectations'] = ''
283         port.expectations_dict = lambda: expectations_dict
284         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
285         capture = OutputCapture()
286         capture.capture_output()
287         exp = TestExpectations(port)
288         _, _, logs = capture.restore_output()
289         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
290
291
292 class ExpectationSyntaxTests(Base):
293     def test_unrecognized_expectation(self):
294         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
295
296     def test_macro(self):
297         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
298         self.parse_exp(exp_str)
299         self.assert_exp('failures/expected/text.html', FAIL)
300
301     def assert_tokenize_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
302         bugs = bugs or []
303         modifiers = modifiers or []
304         expectations = expectations or []
305         warnings = warnings or []
306         filename = 'TestExpectations'
307         line_number = 1
308         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
309         self.assertEquals(expectation_line.warnings, warnings)
310         self.assertEquals(expectation_line.name, name)
311         self.assertEquals(expectation_line.filename, filename)
312         self.assertEquals(expectation_line.line_number, line_number)
313         if not warnings:
314             self.assertEquals(expectation_line.modifiers, modifiers)
315             self.assertEquals(expectation_line.expectations, expectations)
316
317     def test_bare_name(self):
318         self.assert_tokenize_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
319
320     def test_bare_name_and_bugs(self):
321         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
322         self.assert_tokenize_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
323         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
324         self.assert_tokenize_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
325
326     def test_comments(self):
327         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
328         self.assert_tokenize_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
329
330     def test_config_modifiers(self):
331         self.assert_tokenize_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
332         self.assert_tokenize_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
333         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
334
335     def test_unknown_config(self):
336         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
337
338     def test_unknown_expectation(self):
339         self.assert_tokenize_exp('foo.html [ Audio ]', expectations=['Audio'])
340
341     def test_skip(self):
342         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
343
344     def test_slow(self):
345         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
346
347     def test_wontfix(self):
348         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
349
350     def test_blank_line(self):
351         self.assert_tokenize_exp('', name=None)
352
353     def test_warnings(self):
354         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
355         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
356         self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
357
358         self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
359
360
361 class SemanticTests(Base):
362     def test_bug_format(self):
363         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
364
365     def test_bad_bugid(self):
366         try:
367             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
368             self.fail('should have raised an error about a bad bug identifier')
369         except ParseError, exp:
370             self.assertEquals(len(exp.warnings), 1)
371
372     def test_missing_bugid(self):
373         self.parse_exp('failures/expected/text.html [ Failure ]')
374         self.assertFalse(self._exp.has_warnings())
375
376         self._port.warn_if_bug_missing_in_test_expectations = lambda: True
377
378         self.parse_exp('failures/expected/text.html [ Failure ]')
379         line = self._exp._model.get_expectation_line('failures/expected/text.html')
380         self.assertFalse(line.is_invalid())
381         self.assertEquals(line.warnings, ['Test lacks BUG modifier.'])
382
383     def test_skip_and_wontfix(self):
384         # Skip and WontFix are not allowed to have other expectations as well, because those
385         # expectations won't be exercised and may become stale .
386         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
387         self.assertTrue(self._exp.has_warnings())
388
389         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
390         self.assertTrue(self._exp.has_warnings())
391
392         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
393         self.assertTrue(self._exp.has_warnings())
394
395     def test_slow_and_timeout(self):
396         # A test cannot be SLOW and expected to TIMEOUT.
397         self.assertRaises(ParseError, self.parse_exp,
398             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
399
400     def test_rebaseline(self):
401         # Can't lint a file w/ 'REBASELINE' in it.
402         self.assertRaises(ParseError, self.parse_exp,
403             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
404             is_lint_mode=True)
405
406     def test_duplicates(self):
407         self.assertRaises(ParseError, self.parse_exp, """
408 Bug(exp) failures/expected/text.html [ Failure ]
409 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
410
411         self.assertRaises(ParseError, self.parse_exp,
412             self.get_basic_expectations(), overrides="""
413 Bug(override) failures/expected/text.html [ Failure ]
414 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
415
416     def test_missing_file(self):
417         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
418         self.assertTrue(self._exp.has_warnings(), 1)
419
420
421 class PrecedenceTests(Base):
422     def test_file_over_directory(self):
423         # This tests handling precedence of specific lines over directories
424         # and tests expectations covering entire directories.
425         exp_str = """
426 Bug(x) failures/expected/text.html [ Failure ]
427 Bug(y) failures/expected [ WontFix ]
428 """
429         self.parse_exp(exp_str)
430         self.assert_exp('failures/expected/text.html', FAIL)
431         self.assert_exp('failures/expected/crash.html', PASS)
432
433         exp_str = """
434 Bug(x) failures/expected [ WontFix ]
435 Bug(y) failures/expected/text.html [ Failure ]
436 """
437         self.parse_exp(exp_str)
438         self.assert_exp('failures/expected/text.html', FAIL)
439         self.assert_exp('failures/expected/crash.html', PASS)
440
441     def test_ambiguous(self):
442         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
443                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
444
445     def test_more_modifiers(self):
446         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
447                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
448
449     def test_order_in_file(self):
450         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
451                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
452
453     def test_macro_overrides(self):
454         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
455                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
456
457
458 class RemoveConfigurationsTest(Base):
459     def test_remove(self):
460         host = MockHost()
461         test_port = host.port_factory.get('test-win-xp', None)
462         test_port.test_exists = lambda test: True
463         test_port.test_isfile = lambda test: True
464
465         test_config = test_port.test_configuration()
466         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
467 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
468 """}
469         expectations = TestExpectations(test_port, self.get_basic_tests())
470
471         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
472
473         self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
474 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
475 """, actual_expectations)
476
477     def test_remove_line(self):
478         host = MockHost()
479         test_port = host.port_factory.get('test-win-xp', None)
480         test_port.test_exists = lambda test: True
481         test_port.test_isfile = lambda test: True
482
483         test_config = test_port.test_configuration()
484         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
485 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
486 """}
487         expectations = TestExpectations(test_port)
488
489         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
490         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
491         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
492
493         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
494 """, actual_expectations)
495
496
497 class RebaseliningTest(Base):
498     """Test rebaselining-specific functionality."""
499     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
500         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
501         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
502         self.assertEqual(expected_expectations, actual_expectations)
503         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
504         self.assertEqual(expected_overrides, actual_overrides)
505
506     def test_remove(self):
507         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
508                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
509                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
510                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
511                           ['failures/expected/text.html'],
512                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
513                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
514                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
515
516
517     def test_no_get_rebaselining_failures(self):
518         self.parse_exp(self.get_basic_expectations())
519         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
520
521
522 class TestExpectationSerializationTests(unittest.TestCase):
523     def __init__(self, testFunc):
524         host = MockHost()
525         test_port = host.port_factory.get('test-win-xp', None)
526         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
527         unittest.TestCase.__init__(self, testFunc)
528
529     def _tokenize(self, line):
530         return TestExpectationParser._tokenize_line('path', line, 0)
531
532     def assert_round_trip(self, in_string, expected_string=None):
533         expectation = self._tokenize(in_string)
534         if expected_string is None:
535             expected_string = in_string
536         self.assertEqual(expected_string, expectation.to_string(self._converter))
537
538     def assert_list_round_trip(self, in_string, expected_string=None):
539         host = MockHost()
540         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
541         expectations = parser.parse('path', in_string)
542         if expected_string is None:
543             expected_string = in_string
544         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
545
546     def test_unparsed_to_string(self):
547         expectation = TestExpectationLine()
548
549         self.assertEqual(expectation.to_string(self._converter), '')
550         expectation.comment = ' Qux.'
551         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
552         expectation.name = 'bar'
553         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
554         expectation.modifiers = ['foo']
555         # FIXME: case should be preserved here but we can't until we drop the old syntax.
556         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
557         expectation.expectations = ['bAz']
558         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
559         expectation.expectations = ['bAz1', 'baZ2']
560         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
561         expectation.modifiers = ['foo1', 'foO2']
562         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
563         expectation.warnings.append('Oh the horror.')
564         self.assertEqual(expectation.to_string(self._converter), '')
565         expectation.original_string = 'Yes it is!'
566         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
567
568     def test_unparsed_list_to_string(self):
569         expectation = TestExpectationLine()
570         expectation.comment = 'Qux.'
571         expectation.name = 'bar'
572         expectation.modifiers = ['foo']
573         expectation.expectations = ['bAz1', 'baZ2']
574         # FIXME: case should be preserved here but we can't until we drop the old syntax.
575         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
576
577     def test_parsed_to_string(self):
578         expectation_line = TestExpectationLine()
579         expectation_line.parsed_bug_modifiers = ['BUGX']
580         expectation_line.name = 'test/name/for/realz.html'
581         expectation_line.parsed_expectations = set([IMAGE])
582         self.assertEqual(expectation_line.to_string(self._converter), None)
583         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
584         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
585         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
586         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
587
588     def test_serialize_parsed_expectations(self):
589         expectation_line = TestExpectationLine()
590         expectation_line.parsed_expectations = set([])
591         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
592         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
593         expectation_line.parsed_expectations = set([FAIL])
594         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
595         expectation_line.parsed_expectations = set([PASS, IMAGE])
596         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
597         expectation_line.parsed_expectations = set([FAIL, PASS])
598         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
599
600     def test_serialize_parsed_modifier_string(self):
601         expectation_line = TestExpectationLine()
602         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
603         expectation_line.parsed_modifiers = ['for', 'the']
604         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
605         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
606         expectation_line.parsed_bug_modifiers = []
607         expectation_line.parsed_modifiers = []
608         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
609         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
610         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
611         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
612         expectation_line.parsed_bug_modifiers = []
613         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
614         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
615
616     def test_format_line(self):
617         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
618         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
619
620     def test_string_roundtrip(self):
621         self.assert_round_trip('')
622         self.assert_round_trip('FOO')
623         self.assert_round_trip('[')
624         self.assert_round_trip('FOO [')
625         self.assert_round_trip('FOO ] bar')
626         self.assert_round_trip('  FOO [')
627         self.assert_round_trip('    [ FOO ] ')
628         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
629         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
630         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
631         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
632         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
633         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
634         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
635         self.assert_round_trip('FOO ] ] bar ==== BAZ')
636         self.assert_round_trip('=')
637         self.assert_round_trip('#')
638         self.assert_round_trip('# ')
639         self.assert_round_trip('# Foo')
640         self.assert_round_trip('# Foo')
641         self.assert_round_trip('# Foo :')
642         self.assert_round_trip('# Foo : =')
643
644     def test_list_roundtrip(self):
645         self.assert_list_round_trip('')
646         self.assert_list_round_trip('\n')
647         self.assert_list_round_trip('\n\n')
648         self.assert_list_round_trip('bar')
649         self.assert_list_round_trip('bar\n# Qux.')
650         self.assert_list_round_trip('bar\n# Qux.\n')
651
652     def test_reconstitute_only_these(self):
653         lines = []
654         reconstitute_only_these = []
655
656         def add_line(matching_configurations, reconstitute):
657             expectation_line = TestExpectationLine()
658             expectation_line.original_string = "Nay"
659             expectation_line.parsed_bug_modifiers = ['BUGX']
660             expectation_line.name = 'Yay'
661             expectation_line.parsed_expectations = set([IMAGE])
662             expectation_line.matching_configurations = matching_configurations
663             lines.append(expectation_line)
664             if reconstitute:
665                 reconstitute_only_these.append(expectation_line)
666
667         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
668         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
669         serialized = TestExpectations.list_to_string(lines, self._converter)
670         self.assertEquals(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
671         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
672         self.assertEquals(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
673
674     def test_string_whitespace_stripping(self):
675         self.assert_round_trip('\n', '')
676         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
677         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
678         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
679         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
680         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
681         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
682
683
684 if __name__ == '__main__':
685     unittest.main()