Move free functions in test_expectations to TestExpectations class
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38 try:
39     from collections import OrderedDict
40 except ImportError:
41     # Needed for Python < 2.7
42     from webkitpy.thirdparty.ordered_dict import OrderedDict
43
44
45 class MockBugManager(object):
46     def close_bug(self, bug_id, reference_bug_id=None):
47         pass
48
49     def create_bug(self):
50         return "BUG_NEWLY_CREATED"
51
52
53 class Base(unittest.TestCase):
54     # Note that all of these tests are written assuming the configuration
55     # being tested is Windows XP, Release build.
56
57     def __init__(self, testFunc):
58         host = MockHost()
59         self._port = host.port_factory.get('test-win-xp', None)
60         self._exp = None
61         unittest.TestCase.__init__(self, testFunc)
62
63     def get_test(self, test_name):
64         # FIXME: Remove this routine and just reference test names directly.
65         return test_name
66
67     def get_basic_tests(self):
68         return [self.get_test('failures/expected/text.html'),
69                 self.get_test('failures/expected/image_checksum.html'),
70                 self.get_test('failures/expected/crash.html'),
71                 self.get_test('failures/expected/missing_text.html'),
72                 self.get_test('failures/expected/image.html'),
73                 self.get_test('passes/text.html')]
74
75     def get_basic_expectations(self):
76         return """
77 BUG_TEST : failures/expected/text.html = TEXT
78 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
79 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
80 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
81 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
82 """
83
84     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
85         expectations_dict = OrderedDict()
86         expectations_dict['expectations'] = expectations
87         if overrides:
88             expectations_dict['overrides'] = overrides
89         self._port.expectations_dict = lambda: expectations_dict
90         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
91
92     def assert_exp(self, test, result):
93         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
94                           set([result]))
95
96     def assert_bad_expectations(self, expectations, overrides=None):
97         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
98
99     def test_result_was_expected(self):
100         # test basics
101         self.assertEquals(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
102         self.assertEquals(TestExpectations.result_was_expected(TEXT, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
103
104         # test handling of SKIPped tests and results
105         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
106         self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
107
108         # test handling of MISSING results and the REBASELINE modifier
109         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
110         self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
111
112     def test_remove_pixel_failures(self):
113         self.assertEquals(TestExpectations.remove_pixel_failures(set([TEXT])), set([TEXT]))
114         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
115         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
116         self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE_PLUS_TEXT])), set([TEXT]))
117         self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
118
119     def test_suffixes_for_expectations(self):
120         self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT])), set(['txt']))
121         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png']))
122         self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
123         self.assertEquals(TestExpectations.suffixes_for_expectations(set([AUDIO])), set(['wav']))
124         self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT, IMAGE, CRASH])), set(['txt', 'png']))
125         self.assertEquals(TestExpectations.suffixes_for_expectations(set()), set())
126
127
128 class BasicTests(Base):
129     def test_basic(self):
130         self.parse_exp(self.get_basic_expectations())
131         self.assert_exp('failures/expected/text.html', TEXT)
132         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
133         self.assert_exp('passes/text.html', PASS)
134         self.assert_exp('failures/expected/image.html', PASS)
135
136
137 class MiscTests(Base):
138     def test_multiple_results(self):
139         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
140         self.assertEqual(self._exp.get_expectations(
141             self.get_test('failures/expected/text.html')),
142             set([TEXT, CRASH]))
143
144     def test_category_expectations(self):
145         # This test checks unknown tests are not present in the
146         # expectations and that known test part of a test category is
147         # present in the expectations.
148         exp_str = """
149 BUGX WONTFIX : failures/expected = IMAGE
150 """
151         self.parse_exp(exp_str)
152         test_name = 'failures/expected/unknown-test.html'
153         unknown_test = self.get_test(test_name)
154         self.assertRaises(KeyError, self._exp.get_expectations,
155                           unknown_test)
156         self.assert_exp('failures/expected/crash.html', IMAGE)
157
158     def test_get_modifiers(self):
159         self.parse_exp(self.get_basic_expectations())
160         self.assertEqual(self._exp.get_modifiers(
161                          self.get_test('passes/text.html')), [])
162
163     def test_get_expectations_string(self):
164         self.parse_exp(self.get_basic_expectations())
165         self.assertEquals(self._exp.get_expectations_string(
166                           self.get_test('failures/expected/text.html')),
167                           'TEXT')
168
169     def test_expectation_to_string(self):
170         # Normal cases are handled by other tests.
171         self.parse_exp(self.get_basic_expectations())
172         self.assertRaises(ValueError, self._exp.expectation_to_string,
173                           -1)
174
175     def test_get_test_set(self):
176         # Handle some corner cases for this routine not covered by other tests.
177         self.parse_exp(self.get_basic_expectations())
178         s = self._exp.get_test_set(WONTFIX)
179         self.assertEqual(s,
180             set([self.get_test('failures/expected/crash.html'),
181                  self.get_test('failures/expected/image_checksum.html')]))
182         s = self._exp.get_test_set(WONTFIX, CRASH)
183         self.assertEqual(s,
184             set([self.get_test('failures/expected/crash.html')]))
185         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
186         self.assertEqual(s, set([]))
187
188     def test_parse_warning(self):
189         try:
190             self.parse_exp("""FOO : failures/expected/text.html = TEXT
191 SKIP : failures/expected/image.html""", is_lint_mode=True)
192             self.assertFalse(True, "ParseError wasn't raised")
193         except ParseError, e:
194             warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n"
195                         "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
196                         "expectations:2 Missing expectations SKIP : failures/expected/image.html")
197             self.assertEqual(str(e), warnings)
198
199         try:
200             self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
201             self.assertFalse(True, "ParseError wasn't raised")
202         except ParseError, e:
203             warnings = u'expectations:1 Test lacks BUG modifier. failures/expected/text.html'
204             self.assertEqual(str(e), warnings)
205
206     def test_error_on_different_platform(self):
207         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
208         self.assertRaises(ParseError, self.parse_exp,
209             'BUG_TEST MAC : failures/expected/text.html = TEXT\nBUG_TEST MAC : failures/expected/text.html = TEXT',
210             is_lint_mode=True)
211
212     def test_error_on_different_build_type(self):
213         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
214         self.assertRaises(ParseError, self.parse_exp,
215             'BUG_TEST DEBUG : failures/expected/text.html = TEXT\nBUG_TEST DEBUG : failures/expected/text.html = TEXT',
216             is_lint_mode=True)
217
218     def test_overrides(self):
219         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
220                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
221         self.assert_exp('failures/expected/text.html', IMAGE)
222
223     def test_overrides__directory(self):
224         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
225                        "BUG_OVERRIDE: failures/expected = CRASH")
226         self.assert_exp('failures/expected/text.html', CRASH)
227         self.assert_exp('failures/expected/image.html', CRASH)
228
229     def test_overrides__duplicate(self):
230         self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
231                                      "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
232                                      "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
233
234     def test_pixel_tests_flag(self):
235         def match(test, result, pixel_tests_enabled):
236             return self._exp.matches_an_expected_result(
237                 self.get_test(test), result, pixel_tests_enabled)
238
239         self.parse_exp(self.get_basic_expectations())
240         self.assertTrue(match('failures/expected/text.html', TEXT, True))
241         self.assertTrue(match('failures/expected/text.html', TEXT, False))
242         self.assertFalse(match('failures/expected/text.html', CRASH, True))
243         self.assertFalse(match('failures/expected/text.html', CRASH, False))
244         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
245                               True))
246         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
247                               False))
248         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
249         self.assertTrue(match('passes/text.html', PASS, False))
250
251     def test_more_specific_override_resets_skip(self):
252         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
253                        "BUGX : failures/expected/text.html = IMAGE\n")
254         self.assert_exp('failures/expected/text.html', IMAGE)
255         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
256                                                      'failures/expected/text.html') in
257                          self._exp.get_tests_with_result_type(SKIP))
258
259
260 class SkippedTests(Base):
261     def check(self, expectations, overrides, skips, lint=False):
262         port = MockHost().port_factory.get('qt')
263         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
264         expectations_dict = OrderedDict()
265         expectations_dict['expectations'] = expectations
266         if overrides:
267             expectations_dict['overrides'] = overrides
268         port.expectations_dict = lambda: expectations_dict
269         port.skipped_layout_tests = lambda tests: set(skips)
270         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
271
272         # Check that the expectation is for BUG_DUMMY SKIP : ... = PASS
273         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
274                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
275         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
276
277     def test_skipped_tests_work(self):
278         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
279
280     def test_duplicate_skipped_test_fails_lint(self):
281         self.assertRaises(ParseError, self.check, expectations='BUGX : failures/expected/text.html = text\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
282
283     def test_skipped_file_overrides_expectations(self):
284         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
285                    skips=['failures/expected/text.html'])
286
287     def test_skipped_dir_overrides_expectations(self):
288         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
289                    skips=['failures/expected'])
290
291     def test_skipped_file_overrides_overrides(self):
292         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
293                    skips=['failures/expected/text.html'])
294
295     def test_skipped_dir_overrides_overrides(self):
296         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
297                    skips=['failures/expected'])
298
299
300 class ExpectationSyntaxTests(Base):
301     def test_missing_expectation(self):
302         # This is missing the expectation.
303         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
304
305     def test_missing_colon(self):
306         # This is missing the modifiers and the ':'
307         self.assert_bad_expectations('failures/expected/text.html = TEXT')
308
309     def test_too_many_colons(self):
310         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
311
312     def test_too_many_equals_signs(self):
313         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
314
315     def test_unrecognized_expectation(self):
316         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
317
318     def test_macro(self):
319         exp_str = """
320 BUG_TEST WIN : failures/expected/text.html = TEXT
321 """
322         self.parse_exp(exp_str)
323         self.assert_exp('failures/expected/text.html', TEXT)
324
325
326 class SemanticTests(Base):
327     def test_bug_format(self):
328         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
329
330     def test_bad_bugid(self):
331         try:
332             self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
333             self.fail('should have raised an error about a bad bug identifier')
334         except ParseError, exp:
335             self.assertEquals(len(exp.warnings), 1)
336
337     def test_missing_bugid(self):
338         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
339         self.assertTrue(self._exp.has_warnings())
340
341     def test_slow_and_timeout(self):
342         # A test cannot be SLOW and expected to TIMEOUT.
343         self.assertRaises(ParseError, self.parse_exp,
344             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
345
346     def test_rebaseline(self):
347         # Can't lint a file w/ 'REBASELINE' in it.
348         self.assertRaises(ParseError, self.parse_exp,
349             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
350             is_lint_mode=True)
351
352     def test_duplicates(self):
353         self.assertRaises(ParseError, self.parse_exp, """
354 BUG_EXP : failures/expected/text.html = TEXT
355 BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
356
357         self.assertRaises(ParseError, self.parse_exp,
358             self.get_basic_expectations(), overrides="""
359 BUG_OVERRIDE : failures/expected/text.html = TEXT
360 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
361
362     def test_missing_file(self):
363         # This should log a non-fatal error.
364         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
365         self.assertTrue(self._exp.has_warnings(), 1)
366
367
368 class PrecedenceTests(Base):
369     def test_file_over_directory(self):
370         # This tests handling precedence of specific lines over directories
371         # and tests expectations covering entire directories.
372         exp_str = """
373 BUGX : failures/expected/text.html = TEXT
374 BUGX WONTFIX : failures/expected = IMAGE
375 """
376         self.parse_exp(exp_str)
377         self.assert_exp('failures/expected/text.html', TEXT)
378         self.assert_exp('failures/expected/crash.html', IMAGE)
379
380         exp_str = """
381 BUGX WONTFIX : failures/expected = IMAGE
382 BUGX : failures/expected/text.html = TEXT
383 """
384         self.parse_exp(exp_str)
385         self.assert_exp('failures/expected/text.html', TEXT)
386         self.assert_exp('failures/expected/crash.html', IMAGE)
387
388     def test_ambiguous(self):
389         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
390                                      "BUG_TEST WIN : passes/text.html = FAIL\n")
391
392     def test_more_modifiers(self):
393         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
394                                      "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
395
396     def test_order_in_file(self):
397         self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
398                                      "BUG_TEST RELEASE : passes/text.html = PASS\n")
399
400     def test_macro_overrides(self):
401         self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
402                                      "BUG_TEST XP : passes/text.html = TEXT\n")
403
404
405 class RemoveConfigurationsTest(Base):
406     def test_remove(self):
407         host = MockHost()
408         test_port = host.port_factory.get('test-win-xp', None)
409         test_port.test_exists = lambda test: True
410         test_port.test_isfile = lambda test: True
411
412         test_config = test_port.test_configuration()
413         test_port.expectations_dict = lambda: {"expectations": """BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT
414 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
415 """}
416         expectations = TestExpectations(test_port, self.get_basic_tests())
417
418         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
419
420         self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT
421 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
422 """, actual_expectations)
423
424     def test_remove_line(self):
425         host = MockHost()
426         test_port = host.port_factory.get('test-win-xp', None)
427         test_port.test_exists = lambda test: True
428         test_port.test_isfile = lambda test: True
429
430         test_config = test_port.test_configuration()
431         test_port.expectations_dict = lambda: {'expectations': """BUGX WIN RELEASE : failures/expected/foo.html = TEXT
432 BUGY WIN DEBUG : failures/expected/foo.html = CRASH
433 """}
434         expectations = TestExpectations(test_port)
435
436         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
437         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
438         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
439
440         self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH
441 """, actual_expectations)
442
443
444 class RebaseliningTest(Base):
445     """Test rebaselining-specific functionality."""
446     def assertRemove(self, input_expectations, tests, expected_expectations, filename):
447         self.parse_exp(input_expectations, is_lint_mode=False)
448         actual_expectations = self._exp.remove_rebaselined_tests(tests, filename)
449         self.assertEqual(expected_expectations, actual_expectations)
450
451     def test_remove(self):
452         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
453                           'BUGY : failures/expected/image.html = IMAGE\n'
454                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
455                           ['failures/expected/text.html'],
456                           'BUGY : failures/expected/image.html = IMAGE\n'
457                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
458                           'expectations')
459
460         # test that we don't remove lines from the expectations if we're asking for the overrides
461         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
462                           'BUGY : failures/expected/image.html = IMAGE\n'
463                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
464                           ['failures/expected/text.html'],
465                           'BUGX REBASELINE : failures/expected/text.html = TEXT\n'
466                           'BUGY : failures/expected/image.html = IMAGE\n'
467                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
468                           'overrides')
469
470
471     def test_no_get_rebaselining_failures(self):
472         self.parse_exp(self.get_basic_expectations())
473         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
474
475
476 class TestExpectationParserTests(unittest.TestCase):
477     def _tokenize(self, line):
478         return TestExpectationParser._tokenize_line('path', line, 0)
479
480     def test_tokenize_blank(self):
481         expectation = self._tokenize('')
482         self.assertEqual(expectation.comment, None)
483         self.assertEqual(len(expectation.warnings), 0)
484
485     def test_tokenize_missing_colon(self):
486         expectation = self._tokenize('Qux.')
487         self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
488
489     def test_tokenize_extra_colon(self):
490         expectation = self._tokenize('FOO : : bar')
491         self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
492
493     def test_tokenize_empty_comment(self):
494         expectation = self._tokenize('//')
495         self.assertEqual(expectation.comment, '')
496         self.assertEqual(len(expectation.warnings), 0)
497
498     def test_tokenize_comment(self):
499         expectation = self._tokenize('//Qux.')
500         self.assertEqual(expectation.comment, 'Qux.')
501         self.assertEqual(len(expectation.warnings), 0)
502
503     def test_tokenize_missing_equal(self):
504         expectation = self._tokenize('FOO : bar')
505         self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
506
507     def test_tokenize_extra_equal(self):
508         expectation = self._tokenize('FOO : bar = BAZ = Qux.')
509         self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
510
511     def test_tokenize_valid(self):
512         expectation = self._tokenize('FOO : bar = BAZ')
513         self.assertEqual(expectation.comment, None)
514         self.assertEqual(len(expectation.warnings), 0)
515
516     def test_tokenize_valid_with_comment(self):
517         expectation = self._tokenize('FOO : bar = BAZ //Qux.')
518         self.assertEqual(expectation.comment, 'Qux.')
519         self.assertEqual(str(expectation.modifiers), "['FOO']")
520         self.assertEqual(str(expectation.expectations), "['BAZ']")
521         self.assertEqual(len(expectation.warnings), 0)
522
523     def test_tokenize_valid_with_multiple_modifiers(self):
524         expectation = self._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
525         self.assertEqual(expectation.comment, 'Qux.')
526         self.assertEqual(str(expectation.modifiers), "['FOO1', 'FOO2']")
527         self.assertEqual(str(expectation.expectations), "['BAZ']")
528         self.assertEqual(len(expectation.warnings), 0)
529
530     def test_parse_empty_string(self):
531         host = MockHost()
532         test_port = host.port_factory.get('test-win-xp', None)
533         test_port.test_exists = lambda test: True
534         full_test_list = []
535         expectation_line = self._tokenize('')
536         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
537         parser._parse_line(expectation_line)
538         self.assertFalse(expectation_line.is_invalid())
539
540
541 class TestExpectationSerializerTests(unittest.TestCase):
542     def __init__(self, testFunc):
543         host = MockHost()
544         test_port = host.port_factory.get('test-win-xp', None)
545         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
546         self._serializer = TestExpectationSerializer(self._converter)
547         unittest.TestCase.__init__(self, testFunc)
548
549     def _tokenize(self, line):
550         return TestExpectationParser._tokenize_line('path', line, 0)
551
552     def assert_round_trip(self, in_string, expected_string=None):
553         expectation = self._tokenize(in_string)
554         if expected_string is None:
555             expected_string = in_string
556         self.assertEqual(expected_string, self._serializer.to_string(expectation))
557
558     def assert_list_round_trip(self, in_string, expected_string=None):
559         host = MockHost()
560         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
561         expectations = parser.parse('path', in_string)
562         if expected_string is None:
563             expected_string = in_string
564         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, self._converter))
565
566     def test_unparsed_to_string(self):
567         expectation = TestExpectationLine()
568         serializer = TestExpectationSerializer()
569
570         self.assertEqual(serializer.to_string(expectation), '')
571         expectation.comment = 'Qux.'
572         self.assertEqual(serializer.to_string(expectation), '//Qux.')
573         expectation.name = 'bar'
574         self.assertEqual(serializer.to_string(expectation), ' : bar =  //Qux.')
575         expectation.modifiers = ['foo']
576         self.assertEqual(serializer.to_string(expectation), 'FOO : bar =  //Qux.')
577         expectation.expectations = ['bAz']
578         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
579         expectation.expectations = ['bAz1', 'baZ2']
580         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
581         expectation.modifiers = ['foo1', 'foO2']
582         self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
583         expectation.warnings.append('Oh the horror.')
584         self.assertEqual(serializer.to_string(expectation), '')
585         expectation.original_string = 'Yes it is!'
586         self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
587
588     def test_unparsed_list_to_string(self):
589         expectation = TestExpectationLine()
590         expectation.comment = 'Qux.'
591         expectation.name = 'bar'
592         expectation.modifiers = ['foo']
593         expectation.expectations = ['bAz1', 'baZ2']
594         self.assertEqual(TestExpectationSerializer.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
595
596     def test_parsed_to_string(self):
597         expectation_line = TestExpectationLine()
598         expectation_line.parsed_bug_modifiers = ['BUGX']
599         expectation_line.name = 'test/name/for/realz.html'
600         expectation_line.parsed_expectations = set([IMAGE])
601         self.assertEqual(self._serializer.to_string(expectation_line), None)
602         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
603         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
604         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
605         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP : test/name/for/realz.html = IMAGE')
606
607     def test_parsed_expectations_string(self):
608         expectation_line = TestExpectationLine()
609         expectation_line.parsed_expectations = set([])
610         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
611         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
612         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
613         expectation_line.parsed_expectations = set([PASS, IMAGE])
614         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass image')
615         expectation_line.parsed_expectations = set([TEXT, PASS])
616         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass text')
617
618     def test_parsed_modifier_string(self):
619         expectation_line = TestExpectationLine()
620         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
621         expectation_line.parsed_modifiers = ['for', 'the']
622         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
623         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
624         expectation_line.parsed_bug_modifiers = []
625         expectation_line.parsed_modifiers = []
626         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
627         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
628         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
629         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
630         expectation_line.parsed_bug_modifiers = []
631         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
632         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
633
634     def test_format_result(self):
635         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
636         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
637
638     def test_string_roundtrip(self):
639         self.assert_round_trip('')
640         self.assert_round_trip('FOO')
641         self.assert_round_trip(':')
642         self.assert_round_trip('FOO :')
643         self.assert_round_trip('FOO : bar')
644         self.assert_round_trip('  FOO :')
645         self.assert_round_trip('    FOO : bar')
646         self.assert_round_trip('FOO : bar = BAZ')
647         self.assert_round_trip('FOO : bar = BAZ //Qux.')
648         self.assert_round_trip('FOO : bar = BAZ // Qux.')
649         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
650         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
651         self.assert_round_trip('FOO : : bar = BAZ')
652         self.assert_round_trip('FOO : : bar = BAZ')
653         self.assert_round_trip('FOO : : bar ==== BAZ')
654         self.assert_round_trip('=')
655         self.assert_round_trip('//')
656         self.assert_round_trip('// ')
657         self.assert_round_trip('// Foo')
658         self.assert_round_trip('// Foo')
659         self.assert_round_trip('// Foo :')
660         self.assert_round_trip('// Foo : =')
661
662     def test_list_roundtrip(self):
663         self.assert_list_round_trip('')
664         self.assert_list_round_trip('\n')
665         self.assert_list_round_trip('\n\n')
666         self.assert_list_round_trip('bar')
667         self.assert_list_round_trip('bar\n//Qux.')
668         self.assert_list_round_trip('bar\n//Qux.\n')
669
670     def test_reconstitute_only_these(self):
671         lines = []
672         reconstitute_only_these = []
673
674         def add_line(matching_configurations, reconstitute):
675             expectation_line = TestExpectationLine()
676             expectation_line.original_string = "Nay"
677             expectation_line.parsed_bug_modifiers = ['BUGX']
678             expectation_line.name = 'Yay'
679             expectation_line.parsed_expectations = set([IMAGE])
680             expectation_line.matching_configurations = matching_configurations
681             lines.append(expectation_line)
682             if reconstitute:
683                 reconstitute_only_these.append(expectation_line)
684
685         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
686         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
687         serialized = TestExpectationSerializer.list_to_string(lines, self._converter)
688         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nBUGX XP : Yay = IMAGE")
689         serialized = TestExpectationSerializer.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
690         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nNay")
691
692     def test_string_whitespace_stripping(self):
693         self.assert_round_trip('\n', '')
694         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
695         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
696         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
697         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
698         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
699         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
700
701
702 if __name__ == '__main__':
703     unittest.main()