Remove thirdparty/unittest2 from webkitpy
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2013 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.outputcapture import OutputCapture
34
35 from webkitpy.layout_tests.models.test_configuration import *
36 from webkitpy.layout_tests.models.test_expectations import *
37 from webkitpy.layout_tests.models.test_configuration import *
38
39 try:
40     from collections import OrderedDict
41 except ImportError:
42     # Needed for Python < 2.7
43     from webkitpy.thirdparty.ordered_dict import OrderedDict
44
45
46 class Base(unittest.TestCase):
47     # Note that all of these tests are written assuming the configuration
48     # being tested is Windows XP, Release build.
49
50     def __init__(self, testFunc):
51         host = MockHost()
52         self._port = host.port_factory.get('test-win-xp', None)
53         self._exp = None
54         unittest.TestCase.__init__(self, testFunc)
55
56     def get_basic_tests(self):
57         return ['failures/expected/text.html',
58                 'failures/expected/image_checksum.html',
59                 'failures/expected/crash.html',
60                 'failures/expected/missing_text.html',
61                 'failures/expected/image.html',
62                 'passes/text.html']
63
64     def get_basic_expectations(self):
65         return """
66 Bug(test) failures/expected/text.html [ Failure ]
67 Bug(test) failures/expected/crash.html [ WontFix ]
68 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
69 Bug(test) failures/expected/image_checksum.html [ WontFix ]
70 Bug(test) failures/expected/image.html [ WontFix Mac ]
71 """
72
73     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
74         expectations_dict = OrderedDict()
75         expectations_dict['expectations'] = expectations
76         if overrides:
77             expectations_dict['overrides'] = overrides
78         self._port.expectations_dict = lambda: expectations_dict
79         expectations_to_lint = expectations_dict if is_lint_mode else None
80         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
81         self._exp.parse_all_expectations()
82
83     def assert_exp(self, test, result):
84         self.assertEqual(self._exp.model().get_expectations(test),
85                           set([result]))
86
87     def assert_bad_expectations(self, expectations, overrides=None):
88         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
89
90
91 class BasicTests(Base):
92     def test_basic(self):
93         self.parse_exp(self.get_basic_expectations())
94         self.assert_exp('failures/expected/text.html', FAIL)
95         self.assert_exp('failures/expected/image_checksum.html', PASS)
96         self.assert_exp('passes/text.html', PASS)
97         self.assert_exp('failures/expected/image.html', PASS)
98
99
100 class MiscTests(Base):
101     def test_multiple_results(self):
102         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
103         self.assertEqual(self._exp.model().get_expectations(
104             'failures/expected/text.html'),
105             set([FAIL, CRASH]))
106
107     def test_result_was_expected(self):
108         # test basics
109         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
110         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
111
112         # test handling of SKIPped tests and results
113         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
114         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
115
116         # test handling of MISSING results and the REBASELINE modifier
117         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
118         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
119
120     def test_remove_pixel_failures(self):
121         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
122         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
123         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
124         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
125         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
126
127     def test_suffixes_for_expectations(self):
128         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
129         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
130         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
131         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
132
133     def test_category_expectations(self):
134         # This test checks unknown tests are not present in the
135         # expectations and that known test part of a test category is
136         # present in the expectations.
137         exp_str = 'Bug(x) failures/expected [ WontFix ]'
138         self.parse_exp(exp_str)
139         test_name = 'failures/expected/unknown-test.html'
140         unknown_test = test_name
141         self.assertRaises(KeyError, self._exp.model().get_expectations,
142                           unknown_test)
143         self.assert_exp('failures/expected/crash.html', PASS)
144
145     def test_get_modifiers(self):
146         self.parse_exp(self.get_basic_expectations())
147         self.assertEqual(self._exp.model().get_modifiers('passes/text.html'), [])
148
149     def test_get_expectations_string(self):
150         self.parse_exp(self.get_basic_expectations())
151         self.assertEqual(self._exp.model().get_expectations_string('failures/expected/text.html'), 'FAIL')
152
153     def test_expectation_to_string(self):
154         # Normal cases are handled by other tests.
155         self.parse_exp(self.get_basic_expectations())
156         self.assertRaises(ValueError, self._exp.model().expectation_to_string,
157                           -1)
158
159     def test_get_test_set(self):
160         # Handle some corner cases for this routine not covered by other tests.
161         self.parse_exp(self.get_basic_expectations())
162         s = self._exp.model().get_test_set(WONTFIX)
163         self.assertEqual(s,
164             set(['failures/expected/crash.html',
165                  'failures/expected/image_checksum.html']))
166
167     def test_parse_warning(self):
168         try:
169             filesystem = self._port.host.filesystem
170             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
171             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
172                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
173                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
174             self.assertFalse(True, "ParseError wasn't raised")
175         except ParseError, e:
176             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
177                         "expectations:2 Path does not exist. non-existent-test.html")
178             self.assertEqual(str(e), warnings)
179
180     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
181         oc = OutputCapture()
182         try:
183             oc.capture_output()
184             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
185         finally:
186             _, _, logs = oc.restore_output()
187             self.assertNotEquals(logs, '')
188
189     def test_error_on_different_platform(self):
190         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
191         self.assertRaises(ParseError, self.parse_exp,
192             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
193             is_lint_mode=True)
194
195     def test_error_on_different_build_type(self):
196         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
197         self.assertRaises(ParseError, self.parse_exp,
198             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
199             is_lint_mode=True)
200
201     def test_overrides(self):
202         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
203                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
204         self.assert_exp('failures/expected/text.html', IMAGE)
205
206     def test_overrides__directory(self):
207         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
208                        "Bug(override) failures/expected [ Crash ]")
209         self.assert_exp('failures/expected/text.html', CRASH)
210         self.assert_exp('failures/expected/image.html', CRASH)
211
212     def test_overrides__duplicate(self):
213         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
214                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
215                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
216
217     def test_pixel_tests_flag(self):
218         def match(test, result, pixel_tests_enabled):
219             return self._exp.matches_an_expected_result(
220                 test, result, pixel_tests_enabled)
221
222         self.parse_exp(self.get_basic_expectations())
223         self.assertTrue(match('failures/expected/text.html', FAIL, True))
224         self.assertTrue(match('failures/expected/text.html', FAIL, False))
225         self.assertFalse(match('failures/expected/text.html', CRASH, True))
226         self.assertFalse(match('failures/expected/text.html', CRASH, False))
227         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
228                               True))
229         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
230                               False))
231         self.assertTrue(match('failures/expected/crash.html', PASS, False))
232         self.assertTrue(match('passes/text.html', PASS, False))
233
234     def test_more_specific_override_resets_skip(self):
235         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
236                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
237         self.assert_exp('failures/expected/text.html', IMAGE)
238         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
239                                                      'failures/expected/text.html') in
240                          self._exp.model().get_tests_with_result_type(SKIP))
241
242
243 class SkippedTests(Base):
244     def check(self, expectations, overrides, skips, lint=False):
245         port = MockHost().port_factory.get('mac')
246         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
247         expectations_dict = OrderedDict()
248         expectations_dict['expectations'] = expectations
249         if overrides:
250             expectations_dict['overrides'] = overrides
251         port.expectations_dict = lambda: expectations_dict
252         port.skipped_layout_tests = lambda tests: set(skips)
253         expectations_to_lint = expectations_dict if lint else None
254         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
255         exp.parse_all_expectations()
256
257         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
258         self.assertEqual(exp.model().get_modifiers('failures/expected/text.html'),
259                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
260         self.assertEqual(exp.model().get_expectations('failures/expected/text.html'), set([PASS]))
261
262     def test_skipped_tests_work(self):
263         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
264
265     def test_duplicate_skipped_test_fails_lint(self):
266         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
267
268     def test_skipped_file_overrides_expectations(self):
269         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
270                    skips=['failures/expected/text.html'])
271
272     def test_skipped_dir_overrides_expectations(self):
273         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
274                    skips=['failures/expected'])
275
276     def test_skipped_file_overrides_overrides(self):
277         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
278                    skips=['failures/expected/text.html'])
279
280     def test_skipped_dir_overrides_overrides(self):
281         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
282                    skips=['failures/expected'])
283
284     def test_skipped_entry_dont_exist(self):
285         port = MockHost().port_factory.get('mac')
286         expectations_dict = OrderedDict()
287         expectations_dict['expectations'] = ''
288         port.expectations_dict = lambda: expectations_dict
289         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
290         capture = OutputCapture()
291         capture.capture_output()
292         exp = TestExpectations(port)
293         exp.parse_all_expectations()
294         _, _, logs = capture.restore_output()
295         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
296
297
298 class ExpectationSyntaxTests(Base):
299     def test_unrecognized_expectation(self):
300         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
301
302     def test_macro(self):
303         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
304         self.parse_exp(exp_str)
305         self.assert_exp('failures/expected/text.html', FAIL)
306
307     def assert_tokenize_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
308         bugs = bugs or []
309         modifiers = modifiers or []
310         expectations = expectations or []
311         warnings = warnings or []
312         filename = 'TestExpectations'
313         line_number = 1
314         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
315         self.assertEqual(expectation_line.warnings, warnings)
316         self.assertEqual(expectation_line.name, name)
317         self.assertEqual(expectation_line.filename, filename)
318         self.assertEqual(expectation_line.line_number, line_number)
319         if not warnings:
320             self.assertEqual(expectation_line.modifiers, modifiers)
321             self.assertEqual(expectation_line.expectations, expectations)
322
323     def test_bare_name(self):
324         self.assert_tokenize_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
325
326     def test_bare_name_and_bugs(self):
327         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
328         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
329         self.assert_tokenize_exp('webkit.org/b/12345 webkit.org/b/34567 foo.html', modifiers=['BUGWK12345', 'BUGWK34567', 'SKIP'], expectations=['PASS'])
330
331     def test_comments(self):
332         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
333         self.assert_tokenize_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
334
335     def test_config_modifiers(self):
336         self.assert_tokenize_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
337         self.assert_tokenize_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
338         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
339
340     def test_unknown_config(self):
341         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
342
343     def test_unknown_expectation(self):
344         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
345
346     def test_skip(self):
347         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
348
349     def test_slow(self):
350         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
351
352     def test_wontfix(self):
353         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
354         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
355         self.assert_tokenize_exp('foo.html [ WontFix Pass Failure ]', modifiers=['WONTFIX'], expectations=['PASS', 'FAIL'])
356
357     def test_blank_line(self):
358         self.assert_tokenize_exp('', name=None)
359
360     def test_warnings(self):
361         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
362         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
363         self.assert_tokenize_exp('webkit.org/b/12345 ]', warnings=['unexpected "]"'], name=None)
364
365         self.assert_tokenize_exp('foo.html webkit.org/b/12345 ]', warnings=['"webkit.org/b/12345" is not at the start of the line.'])
366
367
368 class SemanticTests(Base):
369     def test_bug_format(self):
370         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
371
372     def test_bad_bugid(self):
373         try:
374             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
375             self.fail('should have raised an error about a bad bug identifier')
376         except ParseError, exp:
377             self.assertEqual(len(exp.warnings), 1)
378
379     def test_missing_bugid(self):
380         self.parse_exp('failures/expected/text.html [ Failure ]')
381         self.assertFalse(self._exp.has_warnings())
382
383         self._port.warn_if_bug_missing_in_test_expectations = lambda: True
384
385         self.parse_exp('failures/expected/text.html [ Failure ]')
386         line = self._exp._model.get_expectation_line('failures/expected/text.html')
387         self.assertFalse(line.is_invalid())
388         self.assertEqual(line.warnings, ['Test lacks BUG modifier.'])
389
390     def test_skip_and_wontfix(self):
391         # Skip is not allowed to have other expectations as well, because those
392         # expectations won't be exercised and may become stale .
393         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
394         self.assertTrue(self._exp.has_warnings())
395
396         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
397         self.assertFalse(self._exp.has_warnings())
398
399         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
400         self.assertFalse(self._exp.has_warnings())
401
402     def test_slow_and_timeout(self):
403         # A test cannot be SLOW and expected to TIMEOUT.
404         self.assertRaises(ParseError, self.parse_exp,
405             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
406
407     def test_rebaseline(self):
408         # Can't lint a file w/ 'REBASELINE' in it.
409         self.assertRaises(ParseError, self.parse_exp,
410             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
411             is_lint_mode=True)
412
413     def test_duplicates(self):
414         self.assertRaises(ParseError, self.parse_exp, """
415 Bug(exp) failures/expected/text.html [ Failure ]
416 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
417
418         self.assertRaises(ParseError, self.parse_exp,
419             self.get_basic_expectations(), overrides="""
420 Bug(override) failures/expected/text.html [ Failure ]
421 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
422
423     def test_missing_file(self):
424         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
425         self.assertTrue(self._exp.has_warnings(), 1)
426
427
428 class PrecedenceTests(Base):
429     def test_file_over_directory(self):
430         # This tests handling precedence of specific lines over directories
431         # and tests expectations covering entire directories.
432         exp_str = """
433 Bug(x) failures/expected/text.html [ Failure ]
434 Bug(y) failures/expected [ WontFix ]
435 """
436         self.parse_exp(exp_str)
437         self.assert_exp('failures/expected/text.html', FAIL)
438         self.assert_exp('failures/expected/crash.html', PASS)
439
440         exp_str = """
441 Bug(x) failures/expected [ WontFix ]
442 Bug(y) failures/expected/text.html [ Failure ]
443 """
444         self.parse_exp(exp_str)
445         self.assert_exp('failures/expected/text.html', FAIL)
446         self.assert_exp('failures/expected/crash.html', PASS)
447
448     def test_ambiguous(self):
449         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
450                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
451
452     def test_more_modifiers(self):
453         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
454                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
455
456     def test_order_in_file(self):
457         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
458                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
459
460     def test_macro_overrides(self):
461         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
462                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
463
464
465 class RemoveConfigurationsTest(Base):
466     def test_remove(self):
467         host = MockHost()
468         test_port = host.port_factory.get('test-win-xp', None)
469         test_port.test_exists = lambda test: True
470         test_port.test_isfile = lambda test: True
471
472         test_config = test_port.test_configuration()
473         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
474 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
475 """}
476         expectations = TestExpectations(test_port, self.get_basic_tests())
477         expectations.parse_all_expectations()
478
479         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
480
481         self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
482 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
483 """, actual_expectations)
484
485     def test_remove_line(self):
486         host = MockHost()
487         test_port = host.port_factory.get('test-win-xp', None)
488         test_port.test_exists = lambda test: True
489         test_port.test_isfile = lambda test: True
490
491         test_config = test_port.test_configuration()
492         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
493 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
494 """}
495         expectations = TestExpectations(test_port)
496         expectations.parse_all_expectations()
497
498         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
499         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
500         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
501
502         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
503 """, actual_expectations)
504
505
506 class RebaseliningTest(Base):
507     """Test rebaselining-specific functionality."""
508     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
509         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
510         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
511         self.assertEqual(expected_expectations, actual_expectations)
512         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
513         self.assertEqual(expected_overrides, actual_overrides)
514
515     def test_remove(self):
516         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
517                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
518                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
519                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
520                           ['failures/expected/text.html'],
521                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
522                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
523                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
524
525         # Ensure that we don't modify unrelated lines, even if we could rewrite them.
526         # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html"
527         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
528                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
529                           'Bug(z) failures/expected/crash.html\n',
530                           '',
531                           ['failures/expected/text.html'],
532                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
533                           'Bug(z) failures/expected/crash.html\n',
534                           '')
535
536     def test_get_rebaselining_failures(self):
537         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
538         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
539         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
540
541         self.parse_exp(self.get_basic_expectations())
542         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
543
544
545 class TestExpectationSerializationTests(unittest.TestCase):
546     def __init__(self, testFunc):
547         host = MockHost()
548         test_port = host.port_factory.get('test-win-xp', None)
549         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
550         unittest.TestCase.__init__(self, testFunc)
551
552     def _tokenize(self, line):
553         return TestExpectationParser._tokenize_line('path', line, 0)
554
555     def assert_round_trip(self, in_string, expected_string=None):
556         expectation = self._tokenize(in_string)
557         if expected_string is None:
558             expected_string = in_string
559         self.assertEqual(expected_string, expectation.to_string(self._converter))
560
561     def assert_list_round_trip(self, in_string, expected_string=None):
562         host = MockHost()
563         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
564         expectations = parser.parse('path', in_string)
565         if expected_string is None:
566             expected_string = in_string
567         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
568
569     def test_unparsed_to_string(self):
570         expectation = TestExpectationLine()
571
572         self.assertEqual(expectation.to_string(self._converter), '')
573         expectation.comment = ' Qux.'
574         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
575         expectation.name = 'bar'
576         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
577         expectation.modifiers = ['foo']
578         # FIXME: case should be preserved here but we can't until we drop the old syntax.
579         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
580         expectation.expectations = ['bAz']
581         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
582         expectation.expectations = ['bAz1', 'baZ2']
583         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
584         expectation.modifiers = ['foo1', 'foO2']
585         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
586         expectation.warnings.append('Oh the horror.')
587         self.assertEqual(expectation.to_string(self._converter), '')
588         expectation.original_string = 'Yes it is!'
589         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
590
591     def test_unparsed_list_to_string(self):
592         expectation = TestExpectationLine()
593         expectation.comment = 'Qux.'
594         expectation.name = 'bar'
595         expectation.modifiers = ['foo']
596         expectation.expectations = ['bAz1', 'baZ2']
597         # FIXME: case should be preserved here but we can't until we drop the old syntax.
598         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
599
600     def test_parsed_to_string(self):
601         expectation_line = TestExpectationLine()
602         expectation_line.parsed_bug_modifiers = ['BUGX']
603         expectation_line.name = 'test/name/for/realz.html'
604         expectation_line.parsed_expectations = set([IMAGE])
605         self.assertEqual(expectation_line.to_string(self._converter), None)
606         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
607         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
608         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
609         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
610
611     def test_serialize_parsed_expectations(self):
612         expectation_line = TestExpectationLine()
613         expectation_line.parsed_expectations = set([])
614         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
615         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
616         expectation_line.parsed_expectations = set([FAIL])
617         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
618         expectation_line.parsed_expectations = set([PASS, IMAGE])
619         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
620         expectation_line.parsed_expectations = set([FAIL, PASS])
621         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
622
623     def test_serialize_parsed_modifier_string(self):
624         expectation_line = TestExpectationLine()
625         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
626         expectation_line.parsed_modifiers = ['for', 'the']
627         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
628         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
629         expectation_line.parsed_bug_modifiers = []
630         expectation_line.parsed_modifiers = []
631         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
632         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
633         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
634         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
635         expectation_line.parsed_bug_modifiers = []
636         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
637         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
638
639     def test_format_line(self):
640         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
641         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
642
643     def test_string_roundtrip(self):
644         self.assert_round_trip('')
645         self.assert_round_trip('FOO')
646         self.assert_round_trip('[')
647         self.assert_round_trip('FOO [')
648         self.assert_round_trip('FOO ] bar')
649         self.assert_round_trip('  FOO [')
650         self.assert_round_trip('    [ FOO ] ')
651         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
652         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
653         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
654         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
655         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
656         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
657         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
658         self.assert_round_trip('FOO ] ] bar ==== BAZ')
659         self.assert_round_trip('=')
660         self.assert_round_trip('#')
661         self.assert_round_trip('# ')
662         self.assert_round_trip('# Foo')
663         self.assert_round_trip('# Foo')
664         self.assert_round_trip('# Foo :')
665         self.assert_round_trip('# Foo : =')
666
667     def test_list_roundtrip(self):
668         self.assert_list_round_trip('')
669         self.assert_list_round_trip('\n')
670         self.assert_list_round_trip('\n\n')
671         self.assert_list_round_trip('bar')
672         self.assert_list_round_trip('bar\n# Qux.')
673         self.assert_list_round_trip('bar\n# Qux.\n')
674
675     def test_reconstitute_only_these(self):
676         lines = []
677         reconstitute_only_these = []
678
679         def add_line(matching_configurations, reconstitute):
680             expectation_line = TestExpectationLine()
681             expectation_line.original_string = "Nay"
682             expectation_line.parsed_bug_modifiers = ['BUGX']
683             expectation_line.name = 'Yay'
684             expectation_line.parsed_expectations = set([IMAGE])
685             expectation_line.matching_configurations = matching_configurations
686             lines.append(expectation_line)
687             if reconstitute:
688                 reconstitute_only_these.append(expectation_line)
689
690         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
691         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
692         serialized = TestExpectations.list_to_string(lines, self._converter)
693         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
694         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
695         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
696
697     def disabled_test_string_whitespace_stripping(self):
698         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
699         self.assert_round_trip('\n', '')
700         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
701         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
702         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
703         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
704         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
705         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')