Use Python 2.7 unittest syntax in our unittests
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest2 as unittest
30
31 from webkitpy.common.host_mock import MockHost
32 from webkitpy.common.system.outputcapture import OutputCapture
33
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38 try:
39     from collections import OrderedDict
40 except ImportError:
41     # Needed for Python < 2.7
42     from webkitpy.thirdparty.ordered_dict import OrderedDict
43
44
45 class Base(unittest.TestCase):
46     # Note that all of these tests are written assuming the configuration
47     # being tested is Windows XP, Release build.
48
49     def __init__(self, testFunc):
50         host = MockHost()
51         self._port = host.port_factory.get('test-win-xp', None)
52         self._exp = None
53         unittest.TestCase.__init__(self, testFunc)
54
55     def get_test(self, test_name):
56         # FIXME: Remove this routine and just reference test names directly.
57         return test_name
58
59     def get_basic_tests(self):
60         return [self.get_test('failures/expected/text.html'),
61                 self.get_test('failures/expected/image_checksum.html'),
62                 self.get_test('failures/expected/crash.html'),
63                 self.get_test('failures/expected/missing_text.html'),
64                 self.get_test('failures/expected/image.html'),
65                 self.get_test('passes/text.html')]
66
67     def get_basic_expectations(self):
68         return """
69 Bug(test) failures/expected/text.html [ Failure ]
70 Bug(test) failures/expected/crash.html [ WontFix ]
71 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
72 Bug(test) failures/expected/image_checksum.html [ WontFix ]
73 Bug(test) failures/expected/image.html [ WontFix Mac ]
74 """
75
76     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
77         expectations_dict = OrderedDict()
78         expectations_dict['expectations'] = expectations
79         if overrides:
80             expectations_dict['overrides'] = overrides
81         self._port.expectations_dict = lambda: expectations_dict
82         expectations_to_lint = expectations_dict if is_lint_mode else None
83         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
84
85     def assert_exp(self, test, result):
86         self.assertEqual(self._exp.get_expectations(self.get_test(test)),
87                           set([result]))
88
89     def assert_bad_expectations(self, expectations, overrides=None):
90         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
91
92
93 class BasicTests(Base):
94     def test_basic(self):
95         self.parse_exp(self.get_basic_expectations())
96         self.assert_exp('failures/expected/text.html', FAIL)
97         self.assert_exp('failures/expected/image_checksum.html', PASS)
98         self.assert_exp('passes/text.html', PASS)
99         self.assert_exp('failures/expected/image.html', PASS)
100
101
102 class MiscTests(Base):
103     def test_multiple_results(self):
104         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
105         self.assertEqual(self._exp.get_expectations(
106             self.get_test('failures/expected/text.html')),
107             set([FAIL, CRASH]))
108
109     def test_result_was_expected(self):
110         # test basics
111         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
112         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
113
114         # test handling of SKIPped tests and results
115         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
116         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
117
118         # test handling of MISSING results and the REBASELINE modifier
119         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
120         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
121
122     def test_remove_pixel_failures(self):
123         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
124         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
125         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
126         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
127         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
128
129     def test_suffixes_for_expectations(self):
130         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
131         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
132         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
133         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
134
135     def test_category_expectations(self):
136         # This test checks unknown tests are not present in the
137         # expectations and that known test part of a test category is
138         # present in the expectations.
139         exp_str = 'Bug(x) failures/expected [ WontFix ]'
140         self.parse_exp(exp_str)
141         test_name = 'failures/expected/unknown-test.html'
142         unknown_test = self.get_test(test_name)
143         self.assertRaises(KeyError, self._exp.get_expectations,
144                           unknown_test)
145         self.assert_exp('failures/expected/crash.html', PASS)
146
147     def test_get_modifiers(self):
148         self.parse_exp(self.get_basic_expectations())
149         self.assertEqual(self._exp.get_modifiers(
150                          self.get_test('passes/text.html')), [])
151
152     def test_get_expectations_string(self):
153         self.parse_exp(self.get_basic_expectations())
154         self.assertEqual(self._exp.get_expectations_string(
155                           self.get_test('failures/expected/text.html')),
156                           'FAIL')
157
158     def test_expectation_to_string(self):
159         # Normal cases are handled by other tests.
160         self.parse_exp(self.get_basic_expectations())
161         self.assertRaises(ValueError, self._exp.expectation_to_string,
162                           -1)
163
164     def test_get_test_set(self):
165         # Handle some corner cases for this routine not covered by other tests.
166         self.parse_exp(self.get_basic_expectations())
167         s = self._exp.get_test_set(WONTFIX)
168         self.assertEqual(s,
169             set([self.get_test('failures/expected/crash.html'),
170                  self.get_test('failures/expected/image_checksum.html')]))
171
172     def test_parse_warning(self):
173         try:
174             filesystem = self._port.host.filesystem
175             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
176             self.get_test('disabled-test.html-disabled'),
177             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
178                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
179                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
180             self.assertFalse(True, "ParseError wasn't raised")
181         except ParseError, e:
182             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
183                         "expectations:2 Path does not exist. non-existent-test.html")
184             self.assertEqual(str(e), warnings)
185
186     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
187         oc = OutputCapture()
188         try:
189             oc.capture_output()
190             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
191         finally:
192             _, _, logs = oc.restore_output()
193             self.assertNotEquals(logs, '')
194
195     def test_error_on_different_platform(self):
196         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
197         self.assertRaises(ParseError, self.parse_exp,
198             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
199             is_lint_mode=True)
200
201     def test_error_on_different_build_type(self):
202         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
203         self.assertRaises(ParseError, self.parse_exp,
204             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
205             is_lint_mode=True)
206
207     def test_overrides(self):
208         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
209                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
210         self.assert_exp('failures/expected/text.html', IMAGE)
211
212     def test_overrides__directory(self):
213         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
214                        "Bug(override) failures/expected [ Crash ]")
215         self.assert_exp('failures/expected/text.html', CRASH)
216         self.assert_exp('failures/expected/image.html', CRASH)
217
218     def test_overrides__duplicate(self):
219         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
220                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
221                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
222
223     def test_pixel_tests_flag(self):
224         def match(test, result, pixel_tests_enabled):
225             return self._exp.matches_an_expected_result(
226                 self.get_test(test), result, pixel_tests_enabled)
227
228         self.parse_exp(self.get_basic_expectations())
229         self.assertTrue(match('failures/expected/text.html', FAIL, True))
230         self.assertTrue(match('failures/expected/text.html', FAIL, False))
231         self.assertFalse(match('failures/expected/text.html', CRASH, True))
232         self.assertFalse(match('failures/expected/text.html', CRASH, False))
233         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
234                               True))
235         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
236                               False))
237         self.assertTrue(match('failures/expected/crash.html', PASS, False))
238         self.assertTrue(match('passes/text.html', PASS, False))
239
240     def test_more_specific_override_resets_skip(self):
241         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
242                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
243         self.assert_exp('failures/expected/text.html', IMAGE)
244         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
245                                                      'failures/expected/text.html') in
246                          self._exp.get_tests_with_result_type(SKIP))
247
248
249 class SkippedTests(Base):
250     def check(self, expectations, overrides, skips, lint=False):
251         port = MockHost().port_factory.get('qt')
252         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
253         expectations_dict = OrderedDict()
254         expectations_dict['expectations'] = expectations
255         if overrides:
256             expectations_dict['overrides'] = overrides
257         port.expectations_dict = lambda: expectations_dict
258         port.skipped_layout_tests = lambda tests: set(skips)
259         expectations_to_lint = expectations_dict if lint else None
260         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
261
262         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
263         self.assertEqual(exp.get_modifiers('failures/expected/text.html'),
264                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
265         self.assertEqual(exp.get_expectations('failures/expected/text.html'), set([PASS]))
266
267     def test_skipped_tests_work(self):
268         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
269
270     def test_duplicate_skipped_test_fails_lint(self):
271         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
272
273     def test_skipped_file_overrides_expectations(self):
274         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
275                    skips=['failures/expected/text.html'])
276
277     def test_skipped_dir_overrides_expectations(self):
278         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
279                    skips=['failures/expected'])
280
281     def test_skipped_file_overrides_overrides(self):
282         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
283                    skips=['failures/expected/text.html'])
284
285     def test_skipped_dir_overrides_overrides(self):
286         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
287                    skips=['failures/expected'])
288
289     def test_skipped_entry_dont_exist(self):
290         port = MockHost().port_factory.get('qt')
291         expectations_dict = OrderedDict()
292         expectations_dict['expectations'] = ''
293         port.expectations_dict = lambda: expectations_dict
294         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
295         capture = OutputCapture()
296         capture.capture_output()
297         exp = TestExpectations(port)
298         _, _, logs = capture.restore_output()
299         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
300
301
302 class ExpectationSyntaxTests(Base):
303     def test_unrecognized_expectation(self):
304         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
305
306     def test_macro(self):
307         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
308         self.parse_exp(exp_str)
309         self.assert_exp('failures/expected/text.html', FAIL)
310
311     def assert_tokenize_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
312         bugs = bugs or []
313         modifiers = modifiers or []
314         expectations = expectations or []
315         warnings = warnings or []
316         filename = 'TestExpectations'
317         line_number = 1
318         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
319         self.assertEqual(expectation_line.warnings, warnings)
320         self.assertEqual(expectation_line.name, name)
321         self.assertEqual(expectation_line.filename, filename)
322         self.assertEqual(expectation_line.line_number, line_number)
323         if not warnings:
324             self.assertEqual(expectation_line.modifiers, modifiers)
325             self.assertEqual(expectation_line.expectations, expectations)
326
327     def test_bare_name(self):
328         self.assert_tokenize_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
329
330     def test_bare_name_and_bugs(self):
331         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
332         self.assert_tokenize_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
333         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
334         self.assert_tokenize_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
335
336     def test_comments(self):
337         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
338         self.assert_tokenize_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
339
340     def test_config_modifiers(self):
341         self.assert_tokenize_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
342         self.assert_tokenize_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
343         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
344
345     def test_unknown_config(self):
346         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
347
348     def test_unknown_expectation(self):
349         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
350
351     def test_skip(self):
352         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
353
354     def test_slow(self):
355         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
356
357     def test_wontfix(self):
358         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
359         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
360         self.assert_tokenize_exp('foo.html [ WontFix Pass Failure ]', modifiers=['WONTFIX'], expectations=['PASS', 'FAIL'])
361
362     def test_blank_line(self):
363         self.assert_tokenize_exp('', name=None)
364
365     def test_warnings(self):
366         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
367         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
368         self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
369
370         self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
371
372
373 class SemanticTests(Base):
374     def test_bug_format(self):
375         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
376
377     def test_bad_bugid(self):
378         try:
379             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
380             self.fail('should have raised an error about a bad bug identifier')
381         except ParseError, exp:
382             self.assertEqual(len(exp.warnings), 1)
383
384     def test_missing_bugid(self):
385         self.parse_exp('failures/expected/text.html [ Failure ]')
386         self.assertFalse(self._exp.has_warnings())
387
388         self._port.warn_if_bug_missing_in_test_expectations = lambda: True
389
390         self.parse_exp('failures/expected/text.html [ Failure ]')
391         line = self._exp._model.get_expectation_line('failures/expected/text.html')
392         self.assertFalse(line.is_invalid())
393         self.assertEqual(line.warnings, ['Test lacks BUG modifier.'])
394
395     def test_skip_and_wontfix(self):
396         # Skip is not allowed to have other expectations as well, because those
397         # expectations won't be exercised and may become stale .
398         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
399         self.assertTrue(self._exp.has_warnings())
400
401         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
402         self.assertFalse(self._exp.has_warnings())
403
404         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
405         self.assertFalse(self._exp.has_warnings())
406
407     def test_slow_and_timeout(self):
408         # A test cannot be SLOW and expected to TIMEOUT.
409         self.assertRaises(ParseError, self.parse_exp,
410             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
411
412     def test_rebaseline(self):
413         # Can't lint a file w/ 'REBASELINE' in it.
414         self.assertRaises(ParseError, self.parse_exp,
415             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
416             is_lint_mode=True)
417
418     def test_duplicates(self):
419         self.assertRaises(ParseError, self.parse_exp, """
420 Bug(exp) failures/expected/text.html [ Failure ]
421 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
422
423         self.assertRaises(ParseError, self.parse_exp,
424             self.get_basic_expectations(), overrides="""
425 Bug(override) failures/expected/text.html [ Failure ]
426 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
427
428     def test_missing_file(self):
429         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
430         self.assertTrue(self._exp.has_warnings(), 1)
431
432
433 class PrecedenceTests(Base):
434     def test_file_over_directory(self):
435         # This tests handling precedence of specific lines over directories
436         # and tests expectations covering entire directories.
437         exp_str = """
438 Bug(x) failures/expected/text.html [ Failure ]
439 Bug(y) failures/expected [ WontFix ]
440 """
441         self.parse_exp(exp_str)
442         self.assert_exp('failures/expected/text.html', FAIL)
443         self.assert_exp('failures/expected/crash.html', PASS)
444
445         exp_str = """
446 Bug(x) failures/expected [ WontFix ]
447 Bug(y) failures/expected/text.html [ Failure ]
448 """
449         self.parse_exp(exp_str)
450         self.assert_exp('failures/expected/text.html', FAIL)
451         self.assert_exp('failures/expected/crash.html', PASS)
452
453     def test_ambiguous(self):
454         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
455                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
456
457     def test_more_modifiers(self):
458         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
459                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
460
461     def test_order_in_file(self):
462         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
463                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
464
465     def test_macro_overrides(self):
466         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
467                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
468
469
470 class RemoveConfigurationsTest(Base):
471     def test_remove(self):
472         host = MockHost()
473         test_port = host.port_factory.get('test-win-xp', None)
474         test_port.test_exists = lambda test: True
475         test_port.test_isfile = lambda test: True
476
477         test_config = test_port.test_configuration()
478         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
479 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
480 """}
481         expectations = TestExpectations(test_port, self.get_basic_tests())
482
483         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
484
485         self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
486 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
487 """, actual_expectations)
488
489     def test_remove_line(self):
490         host = MockHost()
491         test_port = host.port_factory.get('test-win-xp', None)
492         test_port.test_exists = lambda test: True
493         test_port.test_isfile = lambda test: True
494
495         test_config = test_port.test_configuration()
496         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
497 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
498 """}
499         expectations = TestExpectations(test_port)
500
501         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
502         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
503         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
504
505         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
506 """, actual_expectations)
507
508
509 class RebaseliningTest(Base):
510     """Test rebaselining-specific functionality."""
511     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
512         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
513         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
514         self.assertEqual(expected_expectations, actual_expectations)
515         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
516         self.assertEqual(expected_overrides, actual_overrides)
517
518     def test_remove(self):
519         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
520                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
521                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
522                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
523                           ['failures/expected/text.html'],
524                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
525                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
526                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
527
528         # Ensure that we don't modify unrelated lines, even if we could rewrite them.
529         # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html"
530         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
531                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
532                           'Bug(z) failures/expected/crash.html\n',
533                           '',
534                           ['failures/expected/text.html'],
535                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
536                           'Bug(z) failures/expected/crash.html\n',
537                           '')
538
539     def test_get_rebaselining_failures(self):
540         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
541         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
542         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
543
544         self.parse_exp(self.get_basic_expectations())
545         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
546
547
548 class TestExpectationSerializationTests(unittest.TestCase):
549     def __init__(self, testFunc):
550         host = MockHost()
551         test_port = host.port_factory.get('test-win-xp', None)
552         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
553         unittest.TestCase.__init__(self, testFunc)
554
555     def _tokenize(self, line):
556         return TestExpectationParser._tokenize_line('path', line, 0)
557
558     def assert_round_trip(self, in_string, expected_string=None):
559         expectation = self._tokenize(in_string)
560         if expected_string is None:
561             expected_string = in_string
562         self.assertEqual(expected_string, expectation.to_string(self._converter))
563
564     def assert_list_round_trip(self, in_string, expected_string=None):
565         host = MockHost()
566         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
567         expectations = parser.parse('path', in_string)
568         if expected_string is None:
569             expected_string = in_string
570         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
571
572     def test_unparsed_to_string(self):
573         expectation = TestExpectationLine()
574
575         self.assertEqual(expectation.to_string(self._converter), '')
576         expectation.comment = ' Qux.'
577         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
578         expectation.name = 'bar'
579         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
580         expectation.modifiers = ['foo']
581         # FIXME: case should be preserved here but we can't until we drop the old syntax.
582         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
583         expectation.expectations = ['bAz']
584         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
585         expectation.expectations = ['bAz1', 'baZ2']
586         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
587         expectation.modifiers = ['foo1', 'foO2']
588         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
589         expectation.warnings.append('Oh the horror.')
590         self.assertEqual(expectation.to_string(self._converter), '')
591         expectation.original_string = 'Yes it is!'
592         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
593
594     def test_unparsed_list_to_string(self):
595         expectation = TestExpectationLine()
596         expectation.comment = 'Qux.'
597         expectation.name = 'bar'
598         expectation.modifiers = ['foo']
599         expectation.expectations = ['bAz1', 'baZ2']
600         # FIXME: case should be preserved here but we can't until we drop the old syntax.
601         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
602
603     def test_parsed_to_string(self):
604         expectation_line = TestExpectationLine()
605         expectation_line.parsed_bug_modifiers = ['BUGX']
606         expectation_line.name = 'test/name/for/realz.html'
607         expectation_line.parsed_expectations = set([IMAGE])
608         self.assertEqual(expectation_line.to_string(self._converter), None)
609         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
610         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
611         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
612         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
613
614     def test_serialize_parsed_expectations(self):
615         expectation_line = TestExpectationLine()
616         expectation_line.parsed_expectations = set([])
617         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
618         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
619         expectation_line.parsed_expectations = set([FAIL])
620         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
621         expectation_line.parsed_expectations = set([PASS, IMAGE])
622         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
623         expectation_line.parsed_expectations = set([FAIL, PASS])
624         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
625
626     def test_serialize_parsed_modifier_string(self):
627         expectation_line = TestExpectationLine()
628         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
629         expectation_line.parsed_modifiers = ['for', 'the']
630         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
631         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
632         expectation_line.parsed_bug_modifiers = []
633         expectation_line.parsed_modifiers = []
634         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
635         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
636         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
637         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
638         expectation_line.parsed_bug_modifiers = []
639         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
640         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
641
642     def test_format_line(self):
643         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
644         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
645
646     def test_string_roundtrip(self):
647         self.assert_round_trip('')
648         self.assert_round_trip('FOO')
649         self.assert_round_trip('[')
650         self.assert_round_trip('FOO [')
651         self.assert_round_trip('FOO ] bar')
652         self.assert_round_trip('  FOO [')
653         self.assert_round_trip('    [ FOO ] ')
654         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
655         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
656         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
657         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
658         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
659         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
660         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
661         self.assert_round_trip('FOO ] ] bar ==== BAZ')
662         self.assert_round_trip('=')
663         self.assert_round_trip('#')
664         self.assert_round_trip('# ')
665         self.assert_round_trip('# Foo')
666         self.assert_round_trip('# Foo')
667         self.assert_round_trip('# Foo :')
668         self.assert_round_trip('# Foo : =')
669
670     def test_list_roundtrip(self):
671         self.assert_list_round_trip('')
672         self.assert_list_round_trip('\n')
673         self.assert_list_round_trip('\n\n')
674         self.assert_list_round_trip('bar')
675         self.assert_list_round_trip('bar\n# Qux.')
676         self.assert_list_round_trip('bar\n# Qux.\n')
677
678     def test_reconstitute_only_these(self):
679         lines = []
680         reconstitute_only_these = []
681
682         def add_line(matching_configurations, reconstitute):
683             expectation_line = TestExpectationLine()
684             expectation_line.original_string = "Nay"
685             expectation_line.parsed_bug_modifiers = ['BUGX']
686             expectation_line.name = 'Yay'
687             expectation_line.parsed_expectations = set([IMAGE])
688             expectation_line.matching_configurations = matching_configurations
689             lines.append(expectation_line)
690             if reconstitute:
691                 reconstitute_only_these.append(expectation_line)
692
693         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
694         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
695         serialized = TestExpectations.list_to_string(lines, self._converter)
696         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
697         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
698         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
699
700     def disabled_test_string_whitespace_stripping(self):
701         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
702         self.assert_round_trip('\n', '')
703         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
704         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
705         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
706         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
707         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
708         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')