[NRWT] The nrwt should check the contents of the skipped files with --lint-test-files
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.outputcapture import OutputCapture
34
35 from webkitpy.layout_tests.models.test_configuration import *
36 from webkitpy.layout_tests.models.test_expectations import *
37 from webkitpy.layout_tests.models.test_configuration import *
38
39 try:
40     from collections import OrderedDict
41 except ImportError:
42     # Needed for Python < 2.7
43     from webkitpy.thirdparty.ordered_dict import OrderedDict
44
45
46 class MockBugManager(object):
47     def close_bug(self, bug_id, reference_bug_id=None):
48         pass
49
50     def create_bug(self):
51         return "BUG_NEWLY_CREATED"
52
53
54 class Base(unittest.TestCase):
55     # Note that all of these tests are written assuming the configuration
56     # being tested is Windows XP, Release build.
57
58     def __init__(self, testFunc):
59         host = MockHost()
60         self._port = host.port_factory.get('test-win-xp', None)
61         self._exp = None
62         unittest.TestCase.__init__(self, testFunc)
63
64     def get_test(self, test_name):
65         # FIXME: Remove this routine and just reference test names directly.
66         return test_name
67
68     def get_basic_tests(self):
69         return [self.get_test('failures/expected/text.html'),
70                 self.get_test('failures/expected/image_checksum.html'),
71                 self.get_test('failures/expected/crash.html'),
72                 self.get_test('failures/expected/missing_text.html'),
73                 self.get_test('failures/expected/image.html'),
74                 self.get_test('passes/text.html')]
75
76     def get_basic_expectations(self):
77         return """
78 BUG_TEST : failures/expected/text.html = TEXT
79 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
80 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
81 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
82 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
83 """
84
85     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
86         expectations_dict = OrderedDict()
87         expectations_dict['expectations'] = expectations
88         if overrides:
89             expectations_dict['overrides'] = overrides
90         self._port.expectations_dict = lambda: expectations_dict
91         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
92
93     def assert_exp(self, test, result):
94         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
95                           set([result]))
96
97     def assert_bad_expectations(self, expectations, overrides=None):
98         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
99
100     def test_result_was_expected(self):
101         # test basics
102         self.assertEquals(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
103         self.assertEquals(TestExpectations.result_was_expected(TEXT, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
104
105         # test handling of SKIPped tests and results
106         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
107         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
108
109         # test handling of MISSING results and the REBASELINE modifier
110         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
111         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
112
113     def test_remove_pixel_failures(self):
114         self.assertEquals(TestExpectations.remove_pixel_failures(set([TEXT])), set([TEXT]))
115         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
116         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
117         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE_PLUS_TEXT])), set([TEXT]))
118         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
119
120     def test_suffixes_for_expectations(self):
121         self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT])), set(['txt']))
122         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png']))
123         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
124         self.assertEquals(TestExpectations.suffixes_for_expectations(set([AUDIO])), set(['wav']))
125         self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT, IMAGE, CRASH])), set(['txt', 'png']))
126         self.assertEquals(TestExpectations.suffixes_for_expectations(set()), set())
127
128
129 class BasicTests(Base):
130     def test_basic(self):
131         self.parse_exp(self.get_basic_expectations())
132         self.assert_exp('failures/expected/text.html', TEXT)
133         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
134         self.assert_exp('passes/text.html', PASS)
135         self.assert_exp('failures/expected/image.html', PASS)
136
137
138 class MiscTests(Base):
139     def test_multiple_results(self):
140         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
141         self.assertEqual(self._exp.get_expectations(
142             self.get_test('failures/expected/text.html')),
143             set([TEXT, CRASH]))
144
145     def test_category_expectations(self):
146         # This test checks unknown tests are not present in the
147         # expectations and that known test part of a test category is
148         # present in the expectations.
149         exp_str = """
150 BUGX WONTFIX : failures/expected = IMAGE
151 """
152         self.parse_exp(exp_str)
153         test_name = 'failures/expected/unknown-test.html'
154         unknown_test = self.get_test(test_name)
155         self.assertRaises(KeyError, self._exp.get_expectations,
156                           unknown_test)
157         self.assert_exp('failures/expected/crash.html', IMAGE)
158
159     def test_get_modifiers(self):
160         self.parse_exp(self.get_basic_expectations())
161         self.assertEqual(self._exp.get_modifiers(
162                          self.get_test('passes/text.html')), [])
163
164     def test_get_expectations_string(self):
165         self.parse_exp(self.get_basic_expectations())
166         self.assertEquals(self._exp.get_expectations_string(
167                           self.get_test('failures/expected/text.html')),
168                           'TEXT')
169
170     def test_expectation_to_string(self):
171         # Normal cases are handled by other tests.
172         self.parse_exp(self.get_basic_expectations())
173         self.assertRaises(ValueError, self._exp.expectation_to_string,
174                           -1)
175
176     def test_get_test_set(self):
177         # Handle some corner cases for this routine not covered by other tests.
178         self.parse_exp(self.get_basic_expectations())
179         s = self._exp.get_test_set(WONTFIX)
180         self.assertEqual(s,
181             set([self.get_test('failures/expected/crash.html'),
182                  self.get_test('failures/expected/image_checksum.html')]))
183         s = self._exp.get_test_set(WONTFIX, CRASH)
184         self.assertEqual(s,
185             set([self.get_test('failures/expected/crash.html')]))
186         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
187         self.assertEqual(s, set([]))
188
189     def test_parse_warning(self):
190         try:
191             filesystem = self._port.host.filesystem
192             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
193             self.get_test('disabled-test.html-disabled'),
194             self.parse_exp("FOO : failures/expected/text.html = TEXT\n"
195                 "SKIP : failures/expected/image.html\n"
196                 "BUGRNIWA : non-existent-test.html = TEXT\n"
197                 "BUGRNIWA : disabled-test.html-disabled = IMAGE", is_lint_mode=True)
198             self.assertFalse(True, "ParseError wasn't raised")
199         except ParseError, e:
200             warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n"
201                         "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
202                         "expectations:2 Missing expectations SKIP : failures/expected/image.html\n"
203                         "expectations:3 Path does not exist. non-existent-test.html")
204             self.assertEqual(str(e), warnings)
205
206         try:
207             self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
208             self.assertFalse(True, "ParseError wasn't raised")
209         except ParseError, e:
210             warnings = u'expectations:1 Test lacks BUG modifier. failures/expected/text.html'
211             self.assertEqual(str(e), warnings)
212
213     def test_error_on_different_platform(self):
214         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
215         self.assertRaises(ParseError, self.parse_exp,
216             'BUG_TEST MAC : failures/expected/text.html = TEXT\nBUG_TEST MAC : failures/expected/text.html = TEXT',
217             is_lint_mode=True)
218
219     def test_error_on_different_build_type(self):
220         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
221         self.assertRaises(ParseError, self.parse_exp,
222             'BUG_TEST DEBUG : failures/expected/text.html = TEXT\nBUG_TEST DEBUG : failures/expected/text.html = TEXT',
223             is_lint_mode=True)
224
225     def test_overrides(self):
226         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
227                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
228         self.assert_exp('failures/expected/text.html', IMAGE)
229
230     def test_overrides__directory(self):
231         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
232                        "BUG_OVERRIDE: failures/expected = CRASH")
233         self.assert_exp('failures/expected/text.html', CRASH)
234         self.assert_exp('failures/expected/image.html', CRASH)
235
236     def test_overrides__duplicate(self):
237         self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
238                                      "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
239                                      "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
240
241     def test_pixel_tests_flag(self):
242         def match(test, result, pixel_tests_enabled):
243             return self._exp.matches_an_expected_result(
244                 self.get_test(test), result, pixel_tests_enabled)
245
246         self.parse_exp(self.get_basic_expectations())
247         self.assertTrue(match('failures/expected/text.html', TEXT, True))
248         self.assertTrue(match('failures/expected/text.html', TEXT, False))
249         self.assertFalse(match('failures/expected/text.html', CRASH, True))
250         self.assertFalse(match('failures/expected/text.html', CRASH, False))
251         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
252                               True))
253         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
254                               False))
255         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
256         self.assertTrue(match('passes/text.html', PASS, False))
257
258     def test_more_specific_override_resets_skip(self):
259         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
260                        "BUGX : failures/expected/text.html = IMAGE\n")
261         self.assert_exp('failures/expected/text.html', IMAGE)
262         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
263                                                      'failures/expected/text.html') in
264                          self._exp.get_tests_with_result_type(SKIP))
265
266
267 class SkippedTests(Base):
268     def check(self, expectations, overrides, skips, lint=False):
269         port = MockHost().port_factory.get('qt')
270         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
271         expectations_dict = OrderedDict()
272         expectations_dict['expectations'] = expectations
273         if overrides:
274             expectations_dict['overrides'] = overrides
275         port.expectations_dict = lambda: expectations_dict
276         port.skipped_layout_tests = lambda tests: set(skips)
277         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
278
279         # Check that the expectation is for BUG_DUMMY SKIP : ... = PASS
280         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
281                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
282         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
283
284     def test_skipped_tests_work(self):
285         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
286
287     def test_duplicate_skipped_test_fails_lint(self):
288         self.assertRaises(ParseError, self.check, expectations='BUGX : failures/expected/text.html = text\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
289
290     def test_skipped_file_overrides_expectations(self):
291         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
292                    skips=['failures/expected/text.html'])
293
294     def test_skipped_dir_overrides_expectations(self):
295         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
296                    skips=['failures/expected'])
297
298     def test_skipped_file_overrides_overrides(self):
299         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
300                    skips=['failures/expected/text.html'])
301
302     def test_skipped_dir_overrides_overrides(self):
303         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
304                    skips=['failures/expected'])
305
306     def test_skipped_entry_dont_exist(self):
307         port = MockHost().port_factory.get('qt')
308         expectations_dict = OrderedDict()
309         expectations_dict['expectations'] = ''
310         port.expectations_dict = lambda: expectations_dict
311         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
312         capture = OutputCapture()
313         capture.capture_output()
314         exp = TestExpectations(port)
315         _, _, logs = capture.restore_output()
316         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
317
318
319 class ExpectationSyntaxTests(Base):
320     def test_missing_expectation(self):
321         # This is missing the expectation.
322         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
323
324     def test_missing_colon(self):
325         # This is missing the modifiers and the ':'
326         self.assert_bad_expectations('failures/expected/text.html = TEXT')
327
328     def test_too_many_colons(self):
329         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
330
331     def test_too_many_equals_signs(self):
332         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
333
334     def test_unrecognized_expectation(self):
335         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
336
337     def test_macro(self):
338         exp_str = """
339 BUG_TEST WIN : failures/expected/text.html = TEXT
340 """
341         self.parse_exp(exp_str)
342         self.assert_exp('failures/expected/text.html', TEXT)
343
344
345 class SemanticTests(Base):
346     def test_bug_format(self):
347         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
348
349     def test_bad_bugid(self):
350         try:
351             self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
352             self.fail('should have raised an error about a bad bug identifier')
353         except ParseError, exp:
354             self.assertEquals(len(exp.warnings), 1)
355
356     def test_missing_bugid(self):
357         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
358         self.assertTrue(self._exp.has_warnings())
359
360     def test_slow_and_timeout(self):
361         # A test cannot be SLOW and expected to TIMEOUT.
362         self.assertRaises(ParseError, self.parse_exp,
363             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
364
365     def test_rebaseline(self):
366         # Can't lint a file w/ 'REBASELINE' in it.
367         self.assertRaises(ParseError, self.parse_exp,
368             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
369             is_lint_mode=True)
370
371     def test_duplicates(self):
372         self.assertRaises(ParseError, self.parse_exp, """
373 BUG_EXP : failures/expected/text.html = TEXT
374 BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
375
376         self.assertRaises(ParseError, self.parse_exp,
377             self.get_basic_expectations(), overrides="""
378 BUG_OVERRIDE : failures/expected/text.html = TEXT
379 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
380
381     def test_missing_file(self):
382         # This should log a non-fatal error.
383         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
384         self.assertTrue(self._exp.has_warnings(), 1)
385
386
387 class PrecedenceTests(Base):
388     def test_file_over_directory(self):
389         # This tests handling precedence of specific lines over directories
390         # and tests expectations covering entire directories.
391         exp_str = """
392 BUGX : failures/expected/text.html = TEXT
393 BUGX WONTFIX : failures/expected = IMAGE
394 """
395         self.parse_exp(exp_str)
396         self.assert_exp('failures/expected/text.html', TEXT)
397         self.assert_exp('failures/expected/crash.html', IMAGE)
398
399         exp_str = """
400 BUGX WONTFIX : failures/expected = IMAGE
401 BUGX : failures/expected/text.html = TEXT
402 """
403         self.parse_exp(exp_str)
404         self.assert_exp('failures/expected/text.html', TEXT)
405         self.assert_exp('failures/expected/crash.html', IMAGE)
406
407     def test_ambiguous(self):
408         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
409                                      "BUG_TEST WIN : passes/text.html = FAIL\n")
410
411     def test_more_modifiers(self):
412         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
413                                      "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
414
415     def test_order_in_file(self):
416         self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
417                                      "BUG_TEST RELEASE : passes/text.html = PASS\n")
418
419     def test_macro_overrides(self):
420         self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
421                                      "BUG_TEST XP : passes/text.html = TEXT\n")
422
423
424 class RemoveConfigurationsTest(Base):
425     def test_remove(self):
426         host = MockHost()
427         test_port = host.port_factory.get('test-win-xp', None)
428         test_port.test_exists = lambda test: True
429         test_port.test_isfile = lambda test: True
430
431         test_config = test_port.test_configuration()
432         test_port.expectations_dict = lambda: {"expectations": """BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT
433 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
434 """}
435         expectations = TestExpectations(test_port, self.get_basic_tests())
436
437         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
438
439         self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT
440 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
441 """, actual_expectations)
442
443     def test_remove_line(self):
444         host = MockHost()
445         test_port = host.port_factory.get('test-win-xp', None)
446         test_port.test_exists = lambda test: True
447         test_port.test_isfile = lambda test: True
448
449         test_config = test_port.test_configuration()
450         test_port.expectations_dict = lambda: {'expectations': """BUGX WIN RELEASE : failures/expected/foo.html = TEXT
451 BUGY WIN DEBUG : failures/expected/foo.html = CRASH
452 """}
453         expectations = TestExpectations(test_port)
454
455         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
456         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
457         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
458
459         self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH
460 """, actual_expectations)
461
462
463 class RebaseliningTest(Base):
464     """Test rebaselining-specific functionality."""
465     def assertRemove(self, input_expectations, tests, expected_expectations, filename):
466         self.parse_exp(input_expectations, is_lint_mode=False)
467         actual_expectations = self._exp.remove_rebaselined_tests(tests, filename)
468         self.assertEqual(expected_expectations, actual_expectations)
469
470     def test_remove(self):
471         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
472                           'BUGY : failures/expected/image.html = IMAGE\n'
473                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
474                           ['failures/expected/text.html'],
475                           'BUGY : failures/expected/image.html = IMAGE\n'
476                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
477                           'expectations')
478
479         # test that we don't remove lines from the expectations if we're asking for the overrides
480         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
481                           'BUGY : failures/expected/image.html = IMAGE\n'
482                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
483                           ['failures/expected/text.html'],
484                           'BUGX REBASELINE : failures/expected/text.html = TEXT\n'
485                           'BUGY : failures/expected/image.html = IMAGE\n'
486                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
487                           'overrides')
488
489
490     def test_no_get_rebaselining_failures(self):
491         self.parse_exp(self.get_basic_expectations())
492         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
493
494
495 class TestExpectationParserTests(unittest.TestCase):
496     def _tokenize(self, line):
497         return TestExpectationParser._tokenize_line('path', line, 0)
498
499     def test_tokenize_blank(self):
500         expectation = self._tokenize('')
501         self.assertEqual(expectation.comment, None)
502         self.assertEqual(len(expectation.warnings), 0)
503
504     def test_tokenize_missing_colon(self):
505         expectation = self._tokenize('Qux.')
506         self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
507
508     def test_tokenize_extra_colon(self):
509         expectation = self._tokenize('FOO : : bar')
510         self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
511
512     def test_tokenize_empty_comment(self):
513         expectation = self._tokenize('//')
514         self.assertEqual(expectation.comment, '')
515         self.assertEqual(len(expectation.warnings), 0)
516
517     def test_tokenize_comment(self):
518         expectation = self._tokenize('//Qux.')
519         self.assertEqual(expectation.comment, 'Qux.')
520         self.assertEqual(len(expectation.warnings), 0)
521
522     def test_tokenize_missing_equal(self):
523         expectation = self._tokenize('FOO : bar')
524         self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
525
526     def test_tokenize_extra_equal(self):
527         expectation = self._tokenize('FOO : bar = BAZ = Qux.')
528         self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
529
530     def test_tokenize_valid(self):
531         expectation = self._tokenize('FOO : bar = BAZ')
532         self.assertEqual(expectation.comment, None)
533         self.assertEqual(len(expectation.warnings), 0)
534
535     def test_tokenize_valid_with_comment(self):
536         expectation = self._tokenize('FOO : bar = BAZ //Qux.')
537         self.assertEqual(expectation.comment, 'Qux.')
538         self.assertEqual(str(expectation.modifiers), "['FOO']")
539         self.assertEqual(str(expectation.expectations), "['BAZ']")
540         self.assertEqual(len(expectation.warnings), 0)
541
542     def test_tokenize_valid_with_multiple_modifiers(self):
543         expectation = self._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
544         self.assertEqual(expectation.comment, 'Qux.')
545         self.assertEqual(str(expectation.modifiers), "['FOO1', 'FOO2']")
546         self.assertEqual(str(expectation.expectations), "['BAZ']")
547         self.assertEqual(len(expectation.warnings), 0)
548
549     def test_parse_empty_string(self):
550         host = MockHost()
551         test_port = host.port_factory.get('test-win-xp', None)
552         test_port.test_exists = lambda test: True
553         full_test_list = []
554         expectation_line = self._tokenize('')
555         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
556         parser._parse_line(expectation_line)
557         self.assertFalse(expectation_line.is_invalid())
558
559
560 class TestExpectationSerializationTests(unittest.TestCase):
561     def __init__(self, testFunc):
562         host = MockHost()
563         test_port = host.port_factory.get('test-win-xp', None)
564         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
565         unittest.TestCase.__init__(self, testFunc)
566
567     def _tokenize(self, line):
568         return TestExpectationParser._tokenize_line('path', line, 0)
569
570     def assert_round_trip(self, in_string, expected_string=None):
571         expectation = self._tokenize(in_string)
572         if expected_string is None:
573             expected_string = in_string
574         self.assertEqual(expected_string, expectation.to_string(self._converter))
575
576     def assert_list_round_trip(self, in_string, expected_string=None):
577         host = MockHost()
578         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
579         expectations = parser.parse('path', in_string)
580         if expected_string is None:
581             expected_string = in_string
582         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
583
584     def test_unparsed_to_string(self):
585         expectation = TestExpectationLine()
586
587         self.assertEqual(expectation.to_string(self._converter), '')
588         expectation.comment = 'Qux.'
589         self.assertEqual(expectation.to_string(self._converter), '//Qux.')
590         expectation.name = 'bar'
591         self.assertEqual(expectation.to_string(self._converter), ' : bar =  //Qux.')
592         expectation.modifiers = ['foo']
593         self.assertEqual(expectation.to_string(self._converter), 'FOO : bar =  //Qux.')
594         expectation.expectations = ['bAz']
595         self.assertEqual(expectation.to_string(self._converter), 'FOO : bar = BAZ //Qux.')
596         expectation.expectations = ['bAz1', 'baZ2']
597         self.assertEqual(expectation.to_string(self._converter), 'FOO : bar = BAZ1 BAZ2 //Qux.')
598         expectation.modifiers = ['foo1', 'foO2']
599         self.assertEqual(expectation.to_string(self._converter), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
600         expectation.warnings.append('Oh the horror.')
601         self.assertEqual(expectation.to_string(self._converter), '')
602         expectation.original_string = 'Yes it is!'
603         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
604
605     def test_unparsed_list_to_string(self):
606         expectation = TestExpectationLine()
607         expectation.comment = 'Qux.'
608         expectation.name = 'bar'
609         expectation.modifiers = ['foo']
610         expectation.expectations = ['bAz1', 'baZ2']
611         self.assertEqual(TestExpectations.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
612
613     def test_parsed_to_string(self):
614         expectation_line = TestExpectationLine()
615         expectation_line.parsed_bug_modifiers = ['BUGX']
616         expectation_line.name = 'test/name/for/realz.html'
617         expectation_line.parsed_expectations = set([IMAGE])
618         self.assertEqual(expectation_line.to_string(self._converter), None)
619         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
620         self.assertEqual(expectation_line.to_string(self._converter), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
621         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
622         self.assertEqual(expectation_line.to_string(self._converter), 'BUGX XP : test/name/for/realz.html = IMAGE')
623
624     def test_serialize_parsed_expectations(self):
625         expectation_line = TestExpectationLine()
626         expectation_line.parsed_expectations = set([])
627         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
628         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
629         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
630         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'image+text')
631         expectation_line.parsed_expectations = set([PASS, IMAGE])
632         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
633         expectation_line.parsed_expectations = set([TEXT, PASS])
634         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass text')
635
636     def test_serialize_parsed_modifier_string(self):
637         expectation_line = TestExpectationLine()
638         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
639         expectation_line.parsed_modifiers = ['for', 'the']
640         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
641         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
642         expectation_line.parsed_bug_modifiers = []
643         expectation_line.parsed_modifiers = []
644         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
645         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
646         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
647         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
648         expectation_line.parsed_bug_modifiers = []
649         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
650         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
651
652     def test_format_line(self):
653         self.assertEqual(TestExpectationLine._format_line('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
654         self.assertEqual(TestExpectationLine._format_line('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
655
656     def test_string_roundtrip(self):
657         self.assert_round_trip('')
658         self.assert_round_trip('FOO')
659         self.assert_round_trip(':')
660         self.assert_round_trip('FOO :')
661         self.assert_round_trip('FOO : bar')
662         self.assert_round_trip('  FOO :')
663         self.assert_round_trip('    FOO : bar')
664         self.assert_round_trip('FOO : bar = BAZ')
665         self.assert_round_trip('FOO : bar = BAZ //Qux.')
666         self.assert_round_trip('FOO : bar = BAZ // Qux.')
667         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
668         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
669         self.assert_round_trip('FOO : : bar = BAZ')
670         self.assert_round_trip('FOO : : bar = BAZ')
671         self.assert_round_trip('FOO : : bar ==== BAZ')
672         self.assert_round_trip('=')
673         self.assert_round_trip('//')
674         self.assert_round_trip('// ')
675         self.assert_round_trip('// Foo')
676         self.assert_round_trip('// Foo')
677         self.assert_round_trip('// Foo :')
678         self.assert_round_trip('// Foo : =')
679
680     def test_list_roundtrip(self):
681         self.assert_list_round_trip('')
682         self.assert_list_round_trip('\n')
683         self.assert_list_round_trip('\n\n')
684         self.assert_list_round_trip('bar')
685         self.assert_list_round_trip('bar\n//Qux.')
686         self.assert_list_round_trip('bar\n//Qux.\n')
687
688     def test_reconstitute_only_these(self):
689         lines = []
690         reconstitute_only_these = []
691
692         def add_line(matching_configurations, reconstitute):
693             expectation_line = TestExpectationLine()
694             expectation_line.original_string = "Nay"
695             expectation_line.parsed_bug_modifiers = ['BUGX']
696             expectation_line.name = 'Yay'
697             expectation_line.parsed_expectations = set([IMAGE])
698             expectation_line.matching_configurations = matching_configurations
699             lines.append(expectation_line)
700             if reconstitute:
701                 reconstitute_only_these.append(expectation_line)
702
703         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
704         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
705         serialized = TestExpectations.list_to_string(lines, self._converter)
706         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nBUGX XP : Yay = IMAGE")
707         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
708         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nNay")
709
710     def test_string_whitespace_stripping(self):
711         self.assert_round_trip('\n', '')
712         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
713         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
714         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
715         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
716         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
717         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
718
719
720 if __name__ == '__main__':
721     unittest.main()