ea40dfb48a561cd99061dd094d3c08ee2d9a8cc9
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.outputcapture import OutputCapture
34
35 from webkitpy.layout_tests.models.test_configuration import *
36 from webkitpy.layout_tests.models.test_expectations import *
37 from webkitpy.layout_tests.models.test_configuration import *
38
39 try:
40     from collections import OrderedDict
41 except ImportError:
42     # Needed for Python < 2.7
43     from webkitpy.thirdparty.ordered_dict import OrderedDict
44
45
46 class Base(unittest.TestCase):
47     # Note that all of these tests are written assuming the configuration
48     # being tested is Windows XP, Release build.
49
50     def __init__(self, testFunc):
51         host = MockHost()
52         self._port = host.port_factory.get('test-win-xp', None)
53         self._exp = None
54         unittest.TestCase.__init__(self, testFunc)
55
56     def get_test(self, test_name):
57         # FIXME: Remove this routine and just reference test names directly.
58         return test_name
59
60     def get_basic_tests(self):
61         return [self.get_test('failures/expected/text.html'),
62                 self.get_test('failures/expected/image_checksum.html'),
63                 self.get_test('failures/expected/crash.html'),
64                 self.get_test('failures/expected/missing_text.html'),
65                 self.get_test('failures/expected/image.html'),
66                 self.get_test('passes/text.html')]
67
68     def get_basic_expectations(self):
69         return """
70 Bug(test) failures/expected/text.html [ Failure ]
71 Bug(test) failures/expected/crash.html [ WontFix ]
72 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
73 Bug(test) failures/expected/image_checksum.html [ WontFix ]
74 Bug(test) failures/expected/image.html [ WontFix Mac ]
75 """
76
77     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
78         expectations_dict = OrderedDict()
79         expectations_dict['expectations'] = expectations
80         if overrides:
81             expectations_dict['overrides'] = overrides
82         self._port.expectations_dict = lambda: expectations_dict
83         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
84
85     def assert_exp(self, test, result):
86         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
87                           set([result]))
88
89     def assert_bad_expectations(self, expectations, overrides=None):
90         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
91
92
93 class BasicTests(Base):
94     def test_basic(self):
95         self.parse_exp(self.get_basic_expectations())
96         self.assert_exp('failures/expected/text.html', FAIL)
97         self.assert_exp('failures/expected/image_checksum.html', PASS)
98         self.assert_exp('passes/text.html', PASS)
99         self.assert_exp('failures/expected/image.html', PASS)
100
101
102 class MiscTests(Base):
103     def test_multiple_results(self):
104         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
105         self.assertEqual(self._exp.get_expectations(
106             self.get_test('failures/expected/text.html')),
107             set([FAIL, CRASH]))
108
109     def test_result_was_expected(self):
110         # test basics
111         self.assertEquals(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
112         self.assertEquals(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
113
114         # test handling of SKIPped tests and results
115         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
116         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
117
118         # test handling of MISSING results and the REBASELINE modifier
119         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
120         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
121
122     def test_remove_pixel_failures(self):
123         self.assertEquals(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
124         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
125         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
126         self.assertEquals(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
127         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
128
129     def test_suffixes_for_expectations(self):
130         self.assertEquals(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
131         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
132         self.assertEquals(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
133         self.assertEquals(TestExpectations.suffixes_for_expectations(set()), set())
134
135     def test_category_expectations(self):
136         # This test checks unknown tests are not present in the
137         # expectations and that known test part of a test category is
138         # present in the expectations.
139         exp_str = 'Bug(x) failures/expected [ WontFix ]'
140         self.parse_exp(exp_str)
141         test_name = 'failures/expected/unknown-test.html'
142         unknown_test = self.get_test(test_name)
143         self.assertRaises(KeyError, self._exp.get_expectations,
144                           unknown_test)
145         self.assert_exp('failures/expected/crash.html', PASS)
146
147     def test_get_modifiers(self):
148         self.parse_exp(self.get_basic_expectations())
149         self.assertEqual(self._exp.get_modifiers(
150                          self.get_test('passes/text.html')), [])
151
152     def test_get_expectations_string(self):
153         self.parse_exp(self.get_basic_expectations())
154         self.assertEquals(self._exp.get_expectations_string(
155                           self.get_test('failures/expected/text.html')),
156                           'FAIL')
157
158     def test_expectation_to_string(self):
159         # Normal cases are handled by other tests.
160         self.parse_exp(self.get_basic_expectations())
161         self.assertRaises(ValueError, self._exp.expectation_to_string,
162                           -1)
163
164     def test_get_test_set(self):
165         # Handle some corner cases for this routine not covered by other tests.
166         self.parse_exp(self.get_basic_expectations())
167         s = self._exp.get_test_set(WONTFIX)
168         self.assertEqual(s,
169             set([self.get_test('failures/expected/crash.html'),
170                  self.get_test('failures/expected/image_checksum.html')]))
171
172     def test_parse_warning(self):
173         try:
174             filesystem = self._port.host.filesystem
175             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
176             self.get_test('disabled-test.html-disabled'),
177             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
178                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
179                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
180             self.assertFalse(True, "ParseError wasn't raised")
181         except ParseError, e:
182             warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n"
183                         "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
184                         "expectations:2 Path does not exist. non-existent-test.html")
185             self.assertEqual(str(e), warnings)
186
187     def test_error_on_different_platform(self):
188         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
189         self.assertRaises(ParseError, self.parse_exp,
190             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
191             is_lint_mode=True)
192
193     def test_error_on_different_build_type(self):
194         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
195         self.assertRaises(ParseError, self.parse_exp,
196             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
197             is_lint_mode=True)
198
199     def test_overrides(self):
200         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
201                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
202         self.assert_exp('failures/expected/text.html', IMAGE)
203
204     def test_overrides__directory(self):
205         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
206                        "Bug(override) failures/expected [ Crash ]")
207         self.assert_exp('failures/expected/text.html', CRASH)
208         self.assert_exp('failures/expected/image.html', CRASH)
209
210     def test_overrides__duplicate(self):
211         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
212                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
213                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
214
215     def test_pixel_tests_flag(self):
216         def match(test, result, pixel_tests_enabled):
217             return self._exp.matches_an_expected_result(
218                 self.get_test(test), result, pixel_tests_enabled)
219
220         self.parse_exp(self.get_basic_expectations())
221         self.assertTrue(match('failures/expected/text.html', FAIL, True))
222         self.assertTrue(match('failures/expected/text.html', FAIL, False))
223         self.assertFalse(match('failures/expected/text.html', CRASH, True))
224         self.assertFalse(match('failures/expected/text.html', CRASH, False))
225         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
226                               True))
227         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
228                               False))
229         self.assertTrue(match('failures/expected/crash.html', PASS, False))
230         self.assertTrue(match('passes/text.html', PASS, False))
231
232     def test_more_specific_override_resets_skip(self):
233         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
234                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
235         self.assert_exp('failures/expected/text.html', IMAGE)
236         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
237                                                      'failures/expected/text.html') in
238                          self._exp.get_tests_with_result_type(SKIP))
239
240
241 class SkippedTests(Base):
242     def check(self, expectations, overrides, skips, lint=False):
243         port = MockHost().port_factory.get('qt')
244         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
245         expectations_dict = OrderedDict()
246         expectations_dict['expectations'] = expectations
247         if overrides:
248             expectations_dict['overrides'] = overrides
249         port.expectations_dict = lambda: expectations_dict
250         port.skipped_layout_tests = lambda tests: set(skips)
251         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
252
253         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
254         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
255                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
256         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
257
258     def test_skipped_tests_work(self):
259         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
260
261     def test_duplicate_skipped_test_fails_lint(self):
262         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
263
264     def test_skipped_file_overrides_expectations(self):
265         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
266                    skips=['failures/expected/text.html'])
267
268     def test_skipped_dir_overrides_expectations(self):
269         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
270                    skips=['failures/expected'])
271
272     def test_skipped_file_overrides_overrides(self):
273         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
274                    skips=['failures/expected/text.html'])
275
276     def test_skipped_dir_overrides_overrides(self):
277         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
278                    skips=['failures/expected'])
279
280     def test_skipped_entry_dont_exist(self):
281         port = MockHost().port_factory.get('qt')
282         expectations_dict = OrderedDict()
283         expectations_dict['expectations'] = ''
284         port.expectations_dict = lambda: expectations_dict
285         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
286         capture = OutputCapture()
287         capture.capture_output()
288         exp = TestExpectations(port)
289         _, _, logs = capture.restore_output()
290         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
291
292
293 # FIXME: remove these tests when we stop supporting the old syntax.
294
295 class ExpectationSyntaxTests(Base):
296     def disabled_test_missing_expectation(self):
297         # This is missing the expectation.
298         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
299
300     def disabled_test_missing_colon(self):
301         # This is missing the modifiers and the ':'
302         self.assert_bad_expectations('failures/expected/text.html [ Failure ]')
303
304     def disabled_test_too_many_colons(self):
305         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
306
307     def disabled_test_too_many_equals_signs(self):
308         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = FAIL = IMAGE')
309
310     # Keep these two
311     def test_unrecognized_expectation(self):
312         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
313
314     def test_macro(self):
315         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
316         self.parse_exp(exp_str)
317         self.assert_exp('failures/expected/text.html', FAIL)
318
319
320
321 class NewExpectationSyntaxTests(unittest.TestCase):
322     def assert_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
323         bugs = bugs or []
324         modifiers = modifiers or []
325         expectations = expectations or []
326         warnings = warnings or []
327         filename = 'TestExpectations'
328         line_number = 1
329         expectation_line = TestExpectationParser._tokenize_line_using_new_format(filename, line, line_number)
330         self.assertEquals(expectation_line.warnings, warnings)
331         self.assertEquals(expectation_line.name, name)
332         self.assertEquals(expectation_line.filename, filename)
333         self.assertEquals(expectation_line.line_number, line_number)
334         if not warnings:
335             self.assertEquals(expectation_line.modifiers, modifiers)
336             self.assertEquals(expectation_line.expectations, expectations)
337
338     def test_bare_name(self):
339         self.assert_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
340
341     def test_bare_name_and_bugs(self):
342         self.assert_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
343         self.assert_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
344         self.assert_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
345         self.assert_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
346
347     def test_comments(self):
348         self.assert_exp("# comment", name=None, comment="# comment")
349         self.assert_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
350
351     def test_config_modifiers(self):
352         self.assert_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
353         self.assert_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
354         self.assert_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
355
356     def test_unknown_config(self):
357         self.assert_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
358
359     def test_unknown_expectation(self):
360         self.assert_exp('foo.html [ Audio ]', expectations=['Audio'])
361
362     def test_skip(self):
363         self.assert_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
364
365     def test_slow(self):
366         self.assert_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
367
368     def test_wontfix(self):
369         self.assert_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
370
371     def test_blank_line(self):
372         self.assert_exp('', name=None)
373
374     def test_warnings(self):
375         self.assert_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
376         self.assert_exp('[ [', warnings=['unexpected "["'], name=None)
377         self.assert_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
378
379         self.assert_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
380
381
382 class SemanticTests(Base):
383     def test_bug_format(self):
384         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
385
386     def test_bad_bugid(self):
387         try:
388             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
389             self.fail('should have raised an error about a bad bug identifier')
390         except ParseError, exp:
391             self.assertEquals(len(exp.warnings), 1)
392
393     def test_missing_bugid(self):
394         self.parse_exp('failures/expected/text.html [ Failure ]')
395         self.assertTrue(self._exp.has_warnings())
396
397         self.parse_exp('failures/expected/text.html [ Failure ]')
398         line = self._exp._model.get_expectation_line('failures/expected/text.html')
399         self.assertFalse(line.is_invalid())
400         self.assertEquals(line.warnings, ['Test lacks BUG modifier.'])
401
402     def test_skip_and_wontfix(self):
403         # Skip and WontFix are not allowed to have other expectations as well, because those
404         # expectations won't be exercised and may become stale .
405         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
406         self.assertTrue(self._exp.has_warnings())
407
408         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
409         self.assertTrue(self._exp.has_warnings())
410
411         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
412         self.assertTrue(self._exp.has_warnings())
413
414     def test_slow_and_timeout(self):
415         # A test cannot be SLOW and expected to TIMEOUT.
416         self.assertRaises(ParseError, self.parse_exp,
417             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
418
419     def test_rebaseline(self):
420         # Can't lint a file w/ 'REBASELINE' in it.
421         self.assertRaises(ParseError, self.parse_exp,
422             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
423             is_lint_mode=True)
424
425     def test_duplicates(self):
426         self.assertRaises(ParseError, self.parse_exp, """
427 Bug(exp) failures/expected/text.html [ Failure ]
428 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
429
430         self.assertRaises(ParseError, self.parse_exp,
431             self.get_basic_expectations(), overrides="""
432 Bug(override) failures/expected/text.html [ Failure ]
433 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
434
435     def test_missing_file(self):
436         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
437         self.assertTrue(self._exp.has_warnings(), 1)
438
439
440 class PrecedenceTests(Base):
441     def test_file_over_directory(self):
442         # This tests handling precedence of specific lines over directories
443         # and tests expectations covering entire directories.
444         exp_str = """
445 Bug(x) failures/expected/text.html [ Failure ]
446 Bug(y) failures/expected [ WontFix ]
447 """
448         self.parse_exp(exp_str)
449         self.assert_exp('failures/expected/text.html', FAIL)
450         self.assert_exp('failures/expected/crash.html', PASS)
451
452         exp_str = """
453 Bug(x) failures/expected [ WontFix ]
454 Bug(y) failures/expected/text.html [ Failure ]
455 """
456         self.parse_exp(exp_str)
457         self.assert_exp('failures/expected/text.html', FAIL)
458         self.assert_exp('failures/expected/crash.html', PASS)
459
460     def test_ambiguous(self):
461         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
462                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
463
464     def test_more_modifiers(self):
465         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
466                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
467
468     def test_order_in_file(self):
469         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
470                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
471
472     def test_macro_overrides(self):
473         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
474                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
475
476
477 class RemoveConfigurationsTest(Base):
478     def test_remove(self):
479         host = MockHost()
480         test_port = host.port_factory.get('test-win-xp', None)
481         test_port.test_exists = lambda test: True
482         test_port.test_isfile = lambda test: True
483
484         test_config = test_port.test_configuration()
485         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
486 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
487 """}
488         expectations = TestExpectations(test_port, self.get_basic_tests())
489
490         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
491
492         self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
493 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
494 """, actual_expectations)
495
496     def test_remove_line(self):
497         host = MockHost()
498         test_port = host.port_factory.get('test-win-xp', None)
499         test_port.test_exists = lambda test: True
500         test_port.test_isfile = lambda test: True
501
502         test_config = test_port.test_configuration()
503         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
504 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
505 """}
506         expectations = TestExpectations(test_port)
507
508         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
509         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
510         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
511
512         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
513 """, actual_expectations)
514
515
516 class RebaseliningTest(Base):
517     """Test rebaselining-specific functionality."""
518     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
519         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
520         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
521         self.assertEqual(expected_expectations, actual_expectations)
522         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
523         self.assertEqual(expected_overrides, actual_overrides)
524
525     def test_remove(self):
526         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
527                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
528                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
529                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
530                           ['failures/expected/text.html'],
531                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
532                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
533                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
534
535
536     def test_no_get_rebaselining_failures(self):
537         self.parse_exp(self.get_basic_expectations())
538         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
539
540
541 # FIXME: remove most of these tests when we no longer support the old syntax.
542
543 class OldExpectationParserTests(unittest.TestCase):
544     def _tokenize(self, line):
545         return TestExpectationParser._tokenize_line_using_old_format('path', line, 0)
546
547     def test_tokenize_blank(self):
548         expectation = self._tokenize('')
549         self.assertEqual(expectation.comment, None)
550         self.assertEqual(len(expectation.warnings), 0)
551
552     def test_tokenize_extra_colon(self):
553         expectation = self._tokenize('FOO : : bar')
554         self.assertEqual(expectation.warnings, ["Extraneous ':'"])
555
556     def test_tokenize_empty_comment(self):
557         expectation = self._tokenize('//')
558         self.assertEqual(expectation.comment, '')
559         self.assertEqual(len(expectation.warnings), 0)
560
561     def test_tokenize_comment(self):
562         expectation = self._tokenize('//Qux.')
563         self.assertEqual(expectation.comment, 'Qux.')
564         self.assertEqual(len(expectation.warnings), 0)
565
566     def test_tokenize_missing_equal(self):
567         expectation = self._tokenize('FOO : bar')
568         self.assertEqual(expectation.warnings, ['Missing expectations'])
569
570     def test_tokenize_extra_equal(self):
571         expectation = self._tokenize('FOO : bar = BAZ = Qux.')
572         self.assertEqual(expectation.warnings, ["Extraneous '='"])
573
574     def test_tokenize_valid(self):
575         expectation = self._tokenize('FOO : bar = BAZ')
576         self.assertEqual(expectation.comment, None)
577         self.assertEqual(len(expectation.warnings), 0)
578
579     def test_tokenize_valid_with_comment(self):
580         expectation = self._tokenize('FOO : bar = BAZ //Qux.')
581         self.assertEqual(expectation.comment, 'Qux.')
582         self.assertEqual(str(expectation.modifiers), "['FOO']")
583         self.assertEqual(str(expectation.expectations), "['BAZ']")
584         self.assertEqual(len(expectation.warnings), 0)
585
586     def test_tokenize_valid_with_multiple_modifiers(self):
587         expectation = self._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
588         self.assertEqual(expectation.comment, 'Qux.')
589         self.assertEqual(str(expectation.modifiers), "['FOO1', 'FOO2']")
590         self.assertEqual(str(expectation.expectations), "['BAZ']")
591         self.assertEqual(len(expectation.warnings), 0)
592
593     def test_parse_empty_string(self):
594         host = MockHost()
595         test_port = host.port_factory.get('test-win-xp', None)
596         test_port.test_exists = lambda test: True
597         full_test_list = []
598         expectation_line = self._tokenize('')
599         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
600         parser._parse_line(expectation_line)
601         self.assertFalse(expectation_line.is_invalid())
602
603
604 class TestExpectationSerializationTests(unittest.TestCase):
605     def __init__(self, testFunc):
606         host = MockHost()
607         test_port = host.port_factory.get('test-win-xp', None)
608         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
609         unittest.TestCase.__init__(self, testFunc)
610
611     def _tokenize(self, line):
612         return TestExpectationParser._tokenize_line('path', line, 0)
613
614     def assert_round_trip(self, in_string, expected_string=None):
615         expectation = self._tokenize(in_string)
616         if expected_string is None:
617             expected_string = in_string
618         self.assertEqual(expected_string, expectation.to_string(self._converter))
619
620     def assert_list_round_trip(self, in_string, expected_string=None):
621         host = MockHost()
622         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
623         expectations = parser.parse('path', in_string)
624         if expected_string is None:
625             expected_string = in_string
626         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
627
628     def test_unparsed_to_string(self):
629         expectation = TestExpectationLine()
630
631         self.assertEqual(expectation.to_string(self._converter), '')
632         expectation.comment = ' Qux.'
633         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
634         expectation.name = 'bar'
635         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
636         expectation.modifiers = ['foo']
637         # FIXME: case should be preserved here but we can't until we drop the old syntax.
638         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
639         expectation.expectations = ['bAz']
640         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
641         expectation.expectations = ['bAz1', 'baZ2']
642         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
643         expectation.modifiers = ['foo1', 'foO2']
644         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
645         expectation.warnings.append('Oh the horror.')
646         self.assertEqual(expectation.to_string(self._converter), '')
647         expectation.original_string = 'Yes it is!'
648         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
649
650     def test_unparsed_list_to_string(self):
651         expectation = TestExpectationLine()
652         expectation.comment = 'Qux.'
653         expectation.name = 'bar'
654         expectation.modifiers = ['foo']
655         expectation.expectations = ['bAz1', 'baZ2']
656         # FIXME: case should be preserved here but we can't until we drop the old syntax.
657         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
658
659     def test_parsed_to_string(self):
660         expectation_line = TestExpectationLine()
661         expectation_line.parsed_bug_modifiers = ['BUGX']
662         expectation_line.name = 'test/name/for/realz.html'
663         expectation_line.parsed_expectations = set([IMAGE])
664         self.assertEqual(expectation_line.to_string(self._converter), None)
665         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
666         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
667         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
668         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
669
670     def test_serialize_parsed_expectations(self):
671         expectation_line = TestExpectationLine()
672         expectation_line.parsed_expectations = set([])
673         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
674         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
675         expectation_line.parsed_expectations = set([FAIL])
676         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
677         expectation_line.parsed_expectations = set([PASS, IMAGE])
678         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
679         expectation_line.parsed_expectations = set([FAIL, PASS])
680         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
681
682     def test_serialize_parsed_modifier_string(self):
683         expectation_line = TestExpectationLine()
684         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
685         expectation_line.parsed_modifiers = ['for', 'the']
686         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
687         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
688         expectation_line.parsed_bug_modifiers = []
689         expectation_line.parsed_modifiers = []
690         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
691         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
692         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
693         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
694         expectation_line.parsed_bug_modifiers = []
695         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
696         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
697
698     def test_format_line(self):
699         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
700         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
701
702     def test_string_roundtrip(self):
703         self.assert_round_trip('')
704         self.assert_round_trip('FOO')
705         self.assert_round_trip('[')
706         self.assert_round_trip('FOO [')
707         self.assert_round_trip('FOO ] bar')
708         self.assert_round_trip('  FOO [')
709         self.assert_round_trip('    [ FOO ] ')
710         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
711         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
712         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
713         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
714         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
715         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
716         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
717         self.assert_round_trip('FOO ] ] bar ==== BAZ')
718         self.assert_round_trip('=')
719         self.assert_round_trip('#')
720         self.assert_round_trip('# ')
721         self.assert_round_trip('# Foo')
722         self.assert_round_trip('# Foo')
723         self.assert_round_trip('# Foo :')
724         self.assert_round_trip('# Foo : =')
725
726     def test_list_roundtrip(self):
727         self.assert_list_round_trip('')
728         self.assert_list_round_trip('\n')
729         self.assert_list_round_trip('\n\n')
730         self.assert_list_round_trip('bar')
731         self.assert_list_round_trip('bar\n# Qux.')
732         self.assert_list_round_trip('bar\n# Qux.\n')
733
734     def test_reconstitute_only_these(self):
735         lines = []
736         reconstitute_only_these = []
737
738         def add_line(matching_configurations, reconstitute):
739             expectation_line = TestExpectationLine()
740             expectation_line.original_string = "Nay"
741             expectation_line.parsed_bug_modifiers = ['BUGX']
742             expectation_line.name = 'Yay'
743             expectation_line.parsed_expectations = set([IMAGE])
744             expectation_line.matching_configurations = matching_configurations
745             lines.append(expectation_line)
746             if reconstitute:
747                 reconstitute_only_these.append(expectation_line)
748
749         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
750         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
751         serialized = TestExpectations.list_to_string(lines, self._converter)
752         self.assertEquals(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
753         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
754         self.assertEquals(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
755
756     def test_string_whitespace_stripping(self):
757         self.assert_round_trip('\n', '')
758         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
759         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
760         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
761         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
762         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
763         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
764
765
766 if __name__ == '__main__':
767     unittest.main()