Unreviewed, rolling out r110783.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38
39 class MockBugManager(object):
40     def close_bug(self, bug_id, reference_bug_id=None):
41         pass
42
43     def create_bug(self):
44         return "BUG_NEWLY_CREATED"
45
46
47 class FunctionsTest(unittest.TestCase):
48     def test_result_was_expected(self):
49         # test basics
50         self.assertEquals(result_was_expected(PASS, set([PASS]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([PASS]),
53                                               False, False), False)
54
55         # test handling of FAIL expectations
56         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
59                                               False, False), True)
60         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
61                                               False, False), True)
62         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
63                                               False, False), False)
64
65         # test handling of SKIPped tests and results
66         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
67                                               False, True), True)
68         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
69                                               False, False), False)
70
71         # test handling of MISSING results and the REBASELINE modifier
72         self.assertEquals(result_was_expected(MISSING, set([PASS]),
73                                               True, False), True)
74         self.assertEquals(result_was_expected(MISSING, set([PASS]),
75                                               False, False), False)
76
77     def test_remove_pixel_failures(self):
78         self.assertEquals(remove_pixel_failures(set([TEXT])),
79                           set([TEXT]))
80         self.assertEquals(remove_pixel_failures(set([PASS])),
81                           set([PASS]))
82         self.assertEquals(remove_pixel_failures(set([IMAGE])),
83                           set([PASS]))
84         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
87                           set([PASS, CRASH]))
88
89
90 class Base(unittest.TestCase):
91     # Note that all of these tests are written assuming the configuration
92     # being tested is Windows XP, Release build.
93
94     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
95         host = MockHost()
96         self._port = host.port_factory.get('test-win-xp', None)
97         self._exp = None
98         unittest.TestCase.__init__(self, testFunc)
99
100     def get_test(self, test_name):
101         # FIXME: Remove this routine and just reference test names directly.
102         return test_name
103
104     def get_basic_tests(self):
105         return [self.get_test('failures/expected/text.html'),
106                 self.get_test('failures/expected/image_checksum.html'),
107                 self.get_test('failures/expected/crash.html'),
108                 self.get_test('failures/expected/missing_text.html'),
109                 self.get_test('failures/expected/image.html'),
110                 self.get_test('passes/text.html')]
111
112     def get_basic_expectations(self):
113         return """
114 BUG_TEST : failures/expected/text.html = TEXT
115 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
116 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
117 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
118 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
119 """
120
121     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
122         test_config = self._port.test_configuration()
123         self._exp = TestExpectations(self._port,
124              tests=self.get_basic_tests(),
125              expectations=expectations,
126              test_config=test_config,
127              is_lint_mode=is_lint_mode,
128              overrides=overrides)
129
130     def assert_exp(self, test, result):
131         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
132                           set([result]))
133
134     def assert_bad_expectations(self, expectations, overrides=None):
135         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
136
137
138 class BasicTests(Base):
139     def test_basic(self):
140         self.parse_exp(self.get_basic_expectations())
141         self.assert_exp('failures/expected/text.html', TEXT)
142         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
143         self.assert_exp('passes/text.html', PASS)
144         self.assert_exp('failures/expected/image.html', PASS)
145
146
147 class MiscTests(Base):
148     def test_multiple_results(self):
149         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
150         self.assertEqual(self._exp.get_expectations(
151             self.get_test('failures/expected/text.html')),
152             set([TEXT, CRASH]))
153
154     def test_category_expectations(self):
155         # This test checks unknown tests are not present in the
156         # expectations and that known test part of a test category is
157         # present in the expectations.
158         exp_str = """
159 BUGX WONTFIX : failures/expected = IMAGE
160 """
161         self.parse_exp(exp_str)
162         test_name = 'failures/expected/unknown-test.html'
163         unknown_test = self.get_test(test_name)
164         self.assertRaises(KeyError, self._exp.get_expectations,
165                           unknown_test)
166         self.assert_exp('failures/expected/crash.html', IMAGE)
167
168     def test_get_modifiers(self):
169         self.parse_exp(self.get_basic_expectations())
170         self.assertEqual(self._exp.get_modifiers(
171                          self.get_test('passes/text.html')), [])
172
173     def test_get_expectations_string(self):
174         self.parse_exp(self.get_basic_expectations())
175         self.assertEquals(self._exp.get_expectations_string(
176                           self.get_test('failures/expected/text.html')),
177                           'TEXT')
178
179     def test_expectation_to_string(self):
180         # Normal cases are handled by other tests.
181         self.parse_exp(self.get_basic_expectations())
182         self.assertRaises(ValueError, self._exp.expectation_to_string,
183                           -1)
184
185     def test_get_test_set(self):
186         # Handle some corner cases for this routine not covered by other tests.
187         self.parse_exp(self.get_basic_expectations())
188         s = self._exp.get_test_set(WONTFIX)
189         self.assertEqual(s,
190             set([self.get_test('failures/expected/crash.html'),
191                  self.get_test('failures/expected/image_checksum.html')]))
192         s = self._exp.get_test_set(WONTFIX, CRASH)
193         self.assertEqual(s,
194             set([self.get_test('failures/expected/crash.html')]))
195         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
196         self.assertEqual(s, set([]))
197
198     def test_parse_warning(self):
199         try:
200             self.parse_exp("""FOO : failures/expected/text.html = TEXT
201 SKIP : failures/expected/image.html""", is_lint_mode=True)
202             self.assertFalse(True, "ParseError wasn't raised")
203         except ParseError, e:
204             warnings = [u":1 Test lacks BUG modifier. failures/expected/text.html",
205                         u":1 Unrecognized modifier 'foo' failures/expected/text.html",
206                         u":2 Missing expectations SKIP : failures/expected/image.html"]
207             self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(warning) for warning in warnings))
208
209         try:
210             self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
211             self.assertFalse(True, "ParseError wasn't raised")
212         except ParseError, e:
213             warnings = [u':1 Test lacks BUG modifier. failures/expected/text.html']
214             self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(warning) for warning in warnings))
215
216     def test_error_on_different_platform(self):
217         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
218         self.assertRaises(ParseError, self.parse_exp,
219             'BUG_TEST MAC : failures/expected/text.html = TEXT\nBUG_TEST MAC : failures/expected/text.html = TEXT',
220             is_lint_mode=True)
221
222     def test_error_on_different_build_type(self):
223         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
224         self.assertRaises(ParseError, self.parse_exp,
225             'BUG_TEST DEBUG : failures/expected/text.html = TEXT\nBUG_TEST DEBUG : failures/expected/text.html = TEXT',
226             is_lint_mode=True)
227
228     def test_overrides(self):
229         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
230                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
231         self.assert_exp('failures/expected/text.html', IMAGE)
232
233     def test_overrides__duplicate(self):
234         self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
235                                      "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
236                                      "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
237
238     def test_pixel_tests_flag(self):
239         def match(test, result, pixel_tests_enabled):
240             return self._exp.matches_an_expected_result(
241                 self.get_test(test), result, pixel_tests_enabled)
242
243         self.parse_exp(self.get_basic_expectations())
244         self.assertTrue(match('failures/expected/text.html', TEXT, True))
245         self.assertTrue(match('failures/expected/text.html', TEXT, False))
246         self.assertFalse(match('failures/expected/text.html', CRASH, True))
247         self.assertFalse(match('failures/expected/text.html', CRASH, False))
248         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
249                               True))
250         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
251                               False))
252         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
253         self.assertTrue(match('passes/text.html', PASS, False))
254
255     def test_more_specific_override_resets_skip(self):
256         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
257                        "BUGX : failures/expected/text.html = IMAGE\n")
258         self.assert_exp('failures/expected/text.html', IMAGE)
259         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
260                                                      'failures/expected/text.html') in
261                          self._exp.get_tests_with_result_type(SKIP))
262
263     def test_add_skipped_tests(self):
264         port = MockHost().port_factory.get('qt')
265         port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html')] = 'foo'
266         expectations = TestExpectations(port, tests=['failures/expected/text.html'], expectations='', test_config=port.test_configuration(), skipped_tests=set(['failures/expected/text.html']))
267         self.assertEquals(expectations.get_modifiers('failures/expected/text.html'), [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER])
268         self.assertEquals(expectations.get_expectations('failures/expected/text.html'), set([PASS]))
269
270     def test_add_skipped_tests_duplicate(self):
271         port = MockHost().port_factory.get('qt')
272         port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html')] = 'foo'
273         self.assertRaises(ParseError, TestExpectations, port, tests=['failures/expected/text.html'], expectations='BUGX : failures/expected/text.html = text\n', test_config=port.test_configuration(), is_lint_mode=True, skipped_tests=set(['failures/expected/text.html']))
274
275
276 class ExpectationSyntaxTests(Base):
277     def test_missing_expectation(self):
278         # This is missing the expectation.
279         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
280
281     def test_missing_colon(self):
282         # This is missing the modifiers and the ':'
283         self.assert_bad_expectations('failures/expected/text.html = TEXT')
284
285     def test_too_many_colons(self):
286         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
287
288     def test_too_many_equals_signs(self):
289         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
290
291     def test_unrecognized_expectation(self):
292         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
293
294     def test_macro(self):
295         exp_str = """
296 BUG_TEST WIN : failures/expected/text.html = TEXT
297 """
298         self.parse_exp(exp_str)
299         self.assert_exp('failures/expected/text.html', TEXT)
300
301
302 class SemanticTests(Base):
303     def test_bug_format(self):
304         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
305
306     def test_bad_bugid(self):
307         try:
308             self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
309             self.fail('should have raised an error about a bad bug identifier')
310         except ParseError, exp:
311             self.assertEquals(len(exp.warnings), 1)
312
313     def test_missing_bugid(self):
314         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
315         self.assertTrue(self._exp.has_warnings())
316
317     def test_slow_and_timeout(self):
318         # A test cannot be SLOW and expected to TIMEOUT.
319         self.assertRaises(ParseError, self.parse_exp,
320             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
321
322     def test_rebaseline(self):
323         # Can't lint a file w/ 'REBASELINE' in it.
324         self.assertRaises(ParseError, self.parse_exp,
325             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
326             is_lint_mode=True)
327
328     def test_duplicates(self):
329         self.assertRaises(ParseError, self.parse_exp, """
330 BUG_EXP : failures/expected/text.html = TEXT
331 BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
332
333         self.assertRaises(ParseError, self.parse_exp,
334             self.get_basic_expectations(), overrides="""
335 BUG_OVERRIDE : failures/expected/text.html = TEXT
336 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
337
338     def test_missing_file(self):
339         # This should log a non-fatal error.
340         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
341         self.assertTrue(self._exp.has_warnings(), 1)
342
343
344 class PrecedenceTests(Base):
345     def test_file_over_directory(self):
346         # This tests handling precedence of specific lines over directories
347         # and tests expectations covering entire directories.
348         exp_str = """
349 BUGX : failures/expected/text.html = TEXT
350 BUGX WONTFIX : failures/expected = IMAGE
351 """
352         self.parse_exp(exp_str)
353         self.assert_exp('failures/expected/text.html', TEXT)
354         self.assert_exp('failures/expected/crash.html', IMAGE)
355
356         exp_str = """
357 BUGX WONTFIX : failures/expected = IMAGE
358 BUGX : failures/expected/text.html = TEXT
359 """
360         self.parse_exp(exp_str)
361         self.assert_exp('failures/expected/text.html', TEXT)
362         self.assert_exp('failures/expected/crash.html', IMAGE)
363
364     def test_ambiguous(self):
365         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
366                                      "BUG_TEST WIN : passes/text.html = FAIL\n")
367
368     def test_more_modifiers(self):
369         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
370                                      "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
371
372     def test_order_in_file(self):
373         self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
374                                      "BUG_TEST RELEASE : passes/text.html = PASS\n")
375
376     def test_macro_overrides(self):
377         self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
378                                      "BUG_TEST XP : passes/text.html = TEXT\n")
379
380
381 class RebaseliningTest(Base):
382     """Test rebaselining-specific functionality."""
383     def assertRemove(self, input_expectations, tests, expected_expectations):
384         self.parse_exp(input_expectations, is_lint_mode=False)
385         actual_expectations = self._exp.remove_rebaselined_tests(tests)
386         self.assertEqual(expected_expectations, actual_expectations)
387
388     def test_remove(self):
389         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
390                           'BUGY : failures/expected/image.html = IMAGE\n'
391                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
392                           ['failures/expected/text.html'],
393                           'BUGY : failures/expected/image.html = IMAGE\n'
394                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
395
396     def test_no_get_rebaselining_failures(self):
397         self.parse_exp(self.get_basic_expectations())
398         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
399
400
401 class TestExpectationParserTests(unittest.TestCase):
402     def test_tokenize_blank(self):
403         expectation = TestExpectationParser._tokenize('')
404         self.assertEqual(expectation.comment, None)
405         self.assertEqual(len(expectation.warnings), 0)
406
407     def test_tokenize_missing_colon(self):
408         expectation = TestExpectationParser._tokenize('Qux.')
409         self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
410
411     def test_tokenize_extra_colon(self):
412         expectation = TestExpectationParser._tokenize('FOO : : bar')
413         self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
414
415     def test_tokenize_empty_comment(self):
416         expectation = TestExpectationParser._tokenize('//')
417         self.assertEqual(expectation.comment, '')
418         self.assertEqual(len(expectation.warnings), 0)
419
420     def test_tokenize_comment(self):
421         expectation = TestExpectationParser._tokenize('//Qux.')
422         self.assertEqual(expectation.comment, 'Qux.')
423         self.assertEqual(len(expectation.warnings), 0)
424
425     def test_tokenize_missing_equal(self):
426         expectation = TestExpectationParser._tokenize('FOO : bar')
427         self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
428
429     def test_tokenize_extra_equal(self):
430         expectation = TestExpectationParser._tokenize('FOO : bar = BAZ = Qux.')
431         self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
432
433     def test_tokenize_valid(self):
434         expectation = TestExpectationParser._tokenize('FOO : bar = BAZ')
435         self.assertEqual(expectation.comment, None)
436         self.assertEqual(len(expectation.warnings), 0)
437
438     def test_tokenize_valid_with_comment(self):
439         expectation = TestExpectationParser._tokenize('FOO : bar = BAZ //Qux.')
440         self.assertEqual(expectation.comment, 'Qux.')
441         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
442         self.assertEqual(str(expectation.expectations), '[\'baz\']')
443         self.assertEqual(len(expectation.warnings), 0)
444
445     def test_tokenize_valid_with_multiple_modifiers(self):
446         expectation = TestExpectationParser._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
447         self.assertEqual(expectation.comment, 'Qux.')
448         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
449         self.assertEqual(str(expectation.expectations), '[\'baz\']')
450         self.assertEqual(len(expectation.warnings), 0)
451
452     def test_parse_empty_string(self):
453         host = MockHost()
454         test_port = host.port_factory.get('test-win-xp', None)
455         test_port.test_exists = lambda test: True
456         test_config = test_port.test_configuration()
457         full_test_list = []
458         expectation_line = TestExpectationParser._tokenize('')
459         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
460         parser._parse_line(expectation_line)
461         self.assertFalse(expectation_line.is_invalid())
462
463
464 class TestExpectationSerializerTests(unittest.TestCase):
465     def __init__(self, testFunc):
466         host = MockHost()
467         test_port = host.port_factory.get('test-win-xp', None)
468         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
469         self._serializer = TestExpectationSerializer(self._converter)
470         unittest.TestCase.__init__(self, testFunc)
471
472     def assert_round_trip(self, in_string, expected_string=None):
473         expectation = TestExpectationParser._tokenize(in_string)
474         if expected_string is None:
475             expected_string = in_string
476         self.assertEqual(expected_string, self._serializer.to_string(expectation))
477
478     def assert_list_round_trip(self, in_string, expected_string=None):
479         expectations = TestExpectationParser._tokenize_list(in_string)
480         if expected_string is None:
481             expected_string = in_string
482         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, self._converter))
483
484     def test_unparsed_to_string(self):
485         expectation = TestExpectationLine()
486         serializer = TestExpectationSerializer()
487
488         self.assertEqual(serializer.to_string(expectation), '')
489         expectation.comment = 'Qux.'
490         self.assertEqual(serializer.to_string(expectation), '//Qux.')
491         expectation.name = 'bar'
492         self.assertEqual(serializer.to_string(expectation), ' : bar =  //Qux.')
493         expectation.modifiers = ['foo']
494         self.assertEqual(serializer.to_string(expectation), 'FOO : bar =  //Qux.')
495         expectation.expectations = ['bAz']
496         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
497         expectation.expectations = ['bAz1', 'baZ2']
498         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
499         expectation.modifiers = ['foo1', 'foO2']
500         self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
501         expectation.warnings.append('Oh the horror.')
502         self.assertEqual(serializer.to_string(expectation), '')
503         expectation.original_string = 'Yes it is!'
504         self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
505
506     def test_unparsed_list_to_string(self):
507         expectation = TestExpectationLine()
508         expectation.comment = 'Qux.'
509         expectation.name = 'bar'
510         expectation.modifiers = ['foo']
511         expectation.expectations = ['bAz1', 'baZ2']
512         self.assertEqual(TestExpectationSerializer.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
513
514     def test_parsed_to_string(self):
515         expectation_line = TestExpectationLine()
516         expectation_line.parsed_bug_modifiers = ['BUGX']
517         expectation_line.name = 'test/name/for/realz.html'
518         expectation_line.parsed_expectations = set([IMAGE])
519         self.assertEqual(self._serializer.to_string(expectation_line), None)
520         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
521         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
522         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
523         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP : test/name/for/realz.html = IMAGE')
524
525     def test_parsed_expectations_string(self):
526         expectation_line = TestExpectationLine()
527         expectation_line.parsed_expectations = set([])
528         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
529         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
530         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
531         expectation_line.parsed_expectations = set([PASS, FAIL])
532         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
533         expectation_line.parsed_expectations = set([FAIL, PASS])
534         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
535
536     def test_parsed_modifier_string(self):
537         expectation_line = TestExpectationLine()
538         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
539         expectation_line.parsed_modifiers = ['for', 'the']
540         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
541         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
542         expectation_line.parsed_bug_modifiers = []
543         expectation_line.parsed_modifiers = []
544         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
545         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
546         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
547         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
548         expectation_line.parsed_bug_modifiers = []
549         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
550         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
551
552     def test_format_result(self):
553         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
554         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
555
556     def test_string_roundtrip(self):
557         self.assert_round_trip('')
558         self.assert_round_trip('FOO')
559         self.assert_round_trip(':')
560         self.assert_round_trip('FOO :')
561         self.assert_round_trip('FOO : bar')
562         self.assert_round_trip('  FOO :')
563         self.assert_round_trip('    FOO : bar')
564         self.assert_round_trip('FOO : bar = BAZ')
565         self.assert_round_trip('FOO : bar = BAZ //Qux.')
566         self.assert_round_trip('FOO : bar = BAZ // Qux.')
567         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
568         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
569         self.assert_round_trip('FOO : : bar = BAZ')
570         self.assert_round_trip('FOO : : bar = BAZ')
571         self.assert_round_trip('FOO : : bar ==== BAZ')
572         self.assert_round_trip('=')
573         self.assert_round_trip('//')
574         self.assert_round_trip('// ')
575         self.assert_round_trip('// Foo')
576         self.assert_round_trip('// Foo')
577         self.assert_round_trip('// Foo :')
578         self.assert_round_trip('// Foo : =')
579
580     def test_list_roundtrip(self):
581         self.assert_list_round_trip('')
582         self.assert_list_round_trip('\n')
583         self.assert_list_round_trip('\n\n')
584         self.assert_list_round_trip('bar')
585         self.assert_list_round_trip('bar\n//Qux.')
586         self.assert_list_round_trip('bar\n//Qux.\n')
587
588     def test_reconstitute_only_these(self):
589         lines = []
590         reconstitute_only_these = []
591
592         def add_line(matching_configurations, reconstitute):
593             expectation_line = TestExpectationLine()
594             expectation_line.original_string = "Nay"
595             expectation_line.parsed_bug_modifiers = ['BUGX']
596             expectation_line.name = 'Yay'
597             expectation_line.parsed_expectations = set([IMAGE])
598             expectation_line.matching_configurations = matching_configurations
599             lines.append(expectation_line)
600             if reconstitute:
601                 reconstitute_only_these.append(expectation_line)
602
603         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
604         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
605         serialized = TestExpectationSerializer.list_to_string(lines, self._converter)
606         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nBUGX XP : Yay = IMAGE")
607         serialized = TestExpectationSerializer.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
608         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nNay")
609
610     def test_string_whitespace_stripping(self):
611         self.assert_round_trip('\n', '')
612         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
613         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
614         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
615         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
616         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
617         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
618
619
620 if __name__ == '__main__':
621     unittest.main()