Eliminate TestExpectationsFile.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.layout_tests import port
33 from webkitpy.layout_tests.port import base
34 from webkitpy.layout_tests.models.test_expectations import *
35
36 class FunctionsTest(unittest.TestCase):
37     def test_result_was_expected(self):
38         # test basics
39         self.assertEquals(result_was_expected(PASS, set([PASS]),
40                                               False, False), True)
41         self.assertEquals(result_was_expected(TEXT, set([PASS]),
42                                               False, False), False)
43
44         # test handling of FAIL expectations
45         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
46                                               False, False), True)
47         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
48                                               False, False), True)
49         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
50                                               False, False), True)
51         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
52                                               False, False), False)
53
54         # test handling of SKIPped tests and results
55         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
56                                               False, True), True)
57         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
58                                               False, False), False)
59
60         # test handling of MISSING results and the REBASELINE modifier
61         self.assertEquals(result_was_expected(MISSING, set([PASS]),
62                                               True, False), True)
63         self.assertEquals(result_was_expected(MISSING, set([PASS]),
64                                               False, False), False)
65
66     def test_remove_pixel_failures(self):
67         self.assertEquals(remove_pixel_failures(set([TEXT])),
68                           set([TEXT]))
69         self.assertEquals(remove_pixel_failures(set([PASS])),
70                           set([PASS]))
71         self.assertEquals(remove_pixel_failures(set([IMAGE])),
72                           set([PASS]))
73         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
74                           set([TEXT]))
75         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
76                           set([PASS, CRASH]))
77
78
79 class Base(unittest.TestCase):
80     # Note that all of these tests are written assuming the configuration
81     # being tested is Windows XP, Release build.
82
83     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
84         self._port = port.get('test-win-xp', None)
85         self._fs = self._port._filesystem
86         self._exp = None
87         unittest.TestCase.__init__(self, testFunc)
88
89     def get_test(self, test_name):
90         # FIXME: Remove this routine and just reference test names directly.
91         return test_name
92
93     def get_basic_tests(self):
94         return [self.get_test('failures/expected/text.html'),
95                 self.get_test('failures/expected/image_checksum.html'),
96                 self.get_test('failures/expected/crash.html'),
97                 self.get_test('failures/expected/missing_text.html'),
98                 self.get_test('failures/expected/image.html'),
99                 self.get_test('passes/text.html')]
100
101     def get_basic_expectations(self):
102         return """
103 BUG_TEST : failures/expected/text.html = TEXT
104 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
105 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
106 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
107 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
108 """
109
110     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
111         test_config = self._port.test_configuration()
112         self._exp = TestExpectations(self._port,
113              tests=self.get_basic_tests(),
114              expectations=expectations,
115              test_config=test_config,
116              is_lint_mode=is_lint_mode,
117              overrides=overrides)
118
119     def assert_exp(self, test, result):
120         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
121                           set([result]))
122
123
124 class BasicTests(Base):
125     def test_basic(self):
126         self.parse_exp(self.get_basic_expectations())
127         self.assert_exp('failures/expected/text.html', TEXT)
128         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
129         self.assert_exp('passes/text.html', PASS)
130         self.assert_exp('failures/expected/image.html', PASS)
131
132
133 class MiscTests(Base):
134     def test_multiple_results(self):
135         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
136         self.assertEqual(self._exp.get_expectations(
137             self.get_test('failures/expected/text.html')),
138             set([TEXT, CRASH]))
139
140     def test_category_expectations(self):
141         # This test checks unknown tests are not present in the
142         # expectations and that known test part of a test category is
143         # present in the expectations.
144         exp_str = """
145 BUGX WONTFIX : failures/expected = IMAGE
146 """
147         self.parse_exp(exp_str)
148         test_name = 'failures/expected/unknown-test.html'
149         unknown_test = self.get_test(test_name)
150         self.assertRaises(KeyError, self._exp.get_expectations,
151                           unknown_test)
152         self.assert_exp('failures/expected/crash.html', IMAGE)
153
154     def test_get_options(self):
155         self.parse_exp(self.get_basic_expectations())
156         self.assertEqual(self._exp.get_options(
157                          self.get_test('passes/text.html')), [])
158
159     def test_expectations_json_for_all_platforms(self):
160         self.parse_exp(self.get_basic_expectations())
161         json_str = self._exp.get_expectations_json_for_all_platforms()
162         # FIXME: test actual content?
163         self.assertTrue(json_str)
164
165     def test_get_expectations_string(self):
166         self.parse_exp(self.get_basic_expectations())
167         self.assertEquals(self._exp.get_expectations_string(
168                           self.get_test('failures/expected/text.html')),
169                           'TEXT')
170
171     def test_expectation_to_string(self):
172         # Normal cases are handled by other tests.
173         self.parse_exp(self.get_basic_expectations())
174         self.assertRaises(ValueError, self._exp.expectation_to_string,
175                           -1)
176
177     def test_get_test_set(self):
178         # Handle some corner cases for this routine not covered by other tests.
179         self.parse_exp(self.get_basic_expectations())
180         s = self._exp.get_test_set(WONTFIX)
181         self.assertEqual(s,
182             set([self.get_test('failures/expected/crash.html'),
183                  self.get_test('failures/expected/image_checksum.html')]))
184         s = self._exp.get_test_set(WONTFIX, CRASH)
185         self.assertEqual(s,
186             set([self.get_test('failures/expected/crash.html')]))
187         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
188         self.assertEqual(s, set([]))
189
190     def test_parse_error_fatal(self):
191         try:
192             self.parse_exp("""FOO : failures/expected/text.html = TEXT
193 SKIP : failures/expected/image.html""")
194             self.assertFalse(True, "ParseError wasn't raised")
195         except ParseError, e:
196             self.assertTrue(e.fatal)
197             exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html",
198                           u"Line:2 Missing expectations in 'SKIP : failures/expected/image.html'"]
199             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
200             self.assertEqual(e.errors, exp_errors)
201
202     def test_parse_error_nonfatal(self):
203         try:
204             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
205                            is_lint_mode=True)
206             self.assertFalse(True, "ParseError wasn't raised")
207         except ParseError, e:
208             self.assertFalse(e.fatal)
209             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
210             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
211             self.assertEqual(e.errors, exp_errors)
212
213     def test_overrides(self):
214         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
215                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
216         self.assert_exp('failures/expected/text.html', IMAGE)
217
218     def test_overrides__duplicate(self):
219         self.assertRaises(ParseError, self.parse_exp,
220              "BUG_EXP: failures/expected/text.html = TEXT",
221              """
222 BUG_OVERRIDE : failures/expected/text.html = IMAGE
223 BUG_OVERRIDE : failures/expected/text.html = CRASH
224 """)
225
226     def test_pixel_tests_flag(self):
227         def match(test, result, pixel_tests_enabled):
228             return self._exp.matches_an_expected_result(
229                 self.get_test(test), result, pixel_tests_enabled)
230
231         self.parse_exp(self.get_basic_expectations())
232         self.assertTrue(match('failures/expected/text.html', TEXT, True))
233         self.assertTrue(match('failures/expected/text.html', TEXT, False))
234         self.assertFalse(match('failures/expected/text.html', CRASH, True))
235         self.assertFalse(match('failures/expected/text.html', CRASH, False))
236         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
237                               True))
238         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
239                               False))
240         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
241         self.assertTrue(match('passes/text.html', PASS, False))
242
243     def test_more_specific_override_resets_skip(self):
244         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
245                        "BUGX : failures/expected/text.html = IMAGE\n")
246         self.assert_exp('failures/expected/text.html', IMAGE)
247         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
248                                                      'failures/expected/text.html') in
249                          self._exp.get_tests_with_result_type(SKIP))
250
251 class ExpectationSyntaxTests(Base):
252     def test_missing_expectation(self):
253         # This is missing the expectation.
254         self.assertRaises(ParseError, self.parse_exp,
255                           'BUG_TEST: failures/expected/text.html')
256
257     def test_missing_colon(self):
258         # This is missing the modifiers and the ':'
259         self.assertRaises(ParseError, self.parse_exp,
260                           'failures/expected/text.html = TEXT')
261
262     def disabled_test_too_many_colons(self):
263         # FIXME: Enable this test and fix the underlying bug.
264         self.assertRaises(ParseError, self.parse_exp,
265                           'BUG_TEST: failures/expected/text.html = PASS :')
266
267     def test_too_many_equals_signs(self):
268         self.assertRaises(ParseError, self.parse_exp,
269                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
270
271     def test_unrecognized_expectation(self):
272         self.assertRaises(ParseError, self.parse_exp,
273                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
274
275     def test_macro(self):
276         exp_str = """
277 BUG_TEST WIN : failures/expected/text.html = TEXT
278 """
279         self.parse_exp(exp_str)
280         self.assert_exp('failures/expected/text.html', TEXT)
281
282
283 class SemanticTests(Base):
284     def test_bug_format(self):
285         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
286
287     def test_missing_bugid(self):
288         # This should log a non-fatal error.
289         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
290         self.assertEqual(
291             len(self._exp.get_non_fatal_errors()), 1)
292
293     def test_slow_and_timeout(self):
294         # A test cannot be SLOW and expected to TIMEOUT.
295         self.assertRaises(ParseError, self.parse_exp,
296             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
297
298     def test_rebaseline(self):
299         # Can't lint a file w/ 'REBASELINE' in it.
300         self.assertRaises(ParseError, self.parse_exp,
301             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
302             is_lint_mode=True)
303
304     def test_duplicates(self):
305         self.assertRaises(ParseError, self.parse_exp, """
306 BUG_EXP : failures/expected/text.html = TEXT
307 BUG_EXP : failures/expected/text.html = IMAGE""")
308
309         self.assertRaises(ParseError, self.parse_exp,
310             self.get_basic_expectations(), overrides="""
311 BUG_OVERRIDE : failures/expected/text.html = TEXT
312 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
313
314     def test_missing_file(self):
315         # This should log a non-fatal error.
316         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
317         self.assertEqual(len(self._exp.get_non_fatal_errors()), 1)
318
319
320 class PrecedenceTests(Base):
321     def test_file_over_directory(self):
322         # This tests handling precedence of specific lines over directories
323         # and tests expectations covering entire directories.
324         exp_str = """
325 BUGX : failures/expected/text.html = TEXT
326 BUGX WONTFIX : failures/expected = IMAGE
327 """
328         self.parse_exp(exp_str)
329         self.assert_exp('failures/expected/text.html', TEXT)
330         self.assert_exp('failures/expected/crash.html', IMAGE)
331
332         exp_str = """
333 BUGX WONTFIX : failures/expected = IMAGE
334 BUGX : failures/expected/text.html = TEXT
335 """
336         self.parse_exp(exp_str)
337         self.assert_exp('failures/expected/text.html', TEXT)
338         self.assert_exp('failures/expected/crash.html', IMAGE)
339
340     def test_ambiguous(self):
341         self.assertRaises(ParseError, self.parse_exp, """
342 BUG_TEST RELEASE : passes/text.html = PASS
343 BUG_TEST WIN : passes/text.html = FAIL
344 """)
345
346     def test_more_modifiers(self):
347         exp_str = """
348 BUG_TEST RELEASE : passes/text.html = PASS
349 BUG_TEST WIN RELEASE : passes/text.html = TEXT
350 """
351         self.assertRaises(ParseError, self.parse_exp, exp_str)
352
353     def test_order_in_file(self):
354         exp_str = """
355 BUG_TEST WIN RELEASE : passes/text.html = TEXT
356 BUG_TEST RELEASE : passes/text.html = PASS
357 """
358         self.assertRaises(ParseError, self.parse_exp, exp_str)
359
360     def test_macro_overrides(self):
361         exp_str = """
362 BUG_TEST WIN : passes/text.html = PASS
363 BUG_TEST XP : passes/text.html = TEXT
364 """
365         self.assertRaises(ParseError, self.parse_exp, exp_str)
366
367
368 class RebaseliningTest(Base):
369     """Test rebaselining-specific functionality."""
370     def assertRemove(self, input_expectations, tests, expected_expectations):
371         self.parse_exp(input_expectations)
372         actual_expectations = self._exp.remove_rebaselined_tests(tests)
373         self.assertEqual(expected_expectations, actual_expectations)
374
375     def test_remove(self):
376         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
377                           'BUGY : failures/expected/image.html = IMAGE\n'
378                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
379                           ['failures/expected/text.html'],
380                           'BUGY : failures/expected/image.html = IMAGE\n'
381                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
382
383     def test_no_get_rebaselining_failures(self):
384         self.parse_exp(self.get_basic_expectations())
385         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
386
387
388 class ModifierTests(unittest.TestCase):
389     def setUp(self):
390         port_obj = port.get('test-win-xp', None)
391         self.config = port_obj.test_configuration()
392         self.matcher = ModifierMatcher(self.config)
393
394     def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0):
395         matcher = self.matcher
396         if values:
397             matcher = ModifierMatcher(self.FakeTestConfiguration(values))
398         match_result = matcher.match(modifiers)
399         self.assertEqual(len(match_result.warnings), 0)
400         self.assertEqual(len(match_result.errors), num_errors)
401         self.assertEqual(match_result.num_matches, expected_num_matches,
402              'match(%s, %s) returned -> %d, expected %d' %
403              (modifiers, str(self.config.values()),
404               match_result.num_matches, expected_num_matches))
405
406     def test_bad_match_modifier(self):
407         self.match(['foo'], num_errors=1)
408
409     def test_none(self):
410         self.match([], 0)
411
412     def test_one(self):
413         self.match(['xp'], 1)
414         self.match(['win'], 1)
415         self.match(['release'], 1)
416         self.match(['cpu'], 1)
417         self.match(['x86'], 1)
418         self.match(['leopard'], -1)
419         self.match(['gpu'], -1)
420         self.match(['debug'], -1)
421
422     def test_two(self):
423         self.match(['xp', 'release'], 2)
424         self.match(['win7', 'release'], -1)
425         self.match(['win7', 'xp'], 1)
426
427     def test_three(self):
428         self.match(['win7', 'xp', 'release'], 2)
429         self.match(['xp', 'debug', 'x86'], -1)
430         self.match(['xp', 'release', 'x86'], 3)
431         self.match(['xp', 'cpu', 'release'], 3)
432
433     def test_four(self):
434         self.match(['xp', 'release', 'cpu', 'x86'], 4)
435         self.match(['win7', 'xp', 'release', 'cpu'], 3)
436         self.match(['win7', 'xp', 'debug', 'cpu'], -1)
437
438     def test_case_insensitivity(self):
439         self.match(['Win'], num_errors=1)
440         self.match(['WIN'], num_errors=1)
441         self.match(['win'], 1)
442
443     def test_duplicates(self):
444         self.match(['release', 'release'], num_errors=1)
445         self.match(['win', 'xp'], num_errors=1)
446         self.match(['xp', 'xp'], num_errors=1)
447         self.match(['xp', 'release', 'xp', 'release'], num_errors=2)
448         self.match(['rebaseline', 'rebaseline'], num_errors=1)
449
450     def test_unknown_option(self):
451         self.match(['vms'], num_errors=1)
452
453     def test_duplicate_bugs(self):
454         # BUG* regexes can appear multiple times.
455         self.match(['bugfoo', 'bugbar'], 0)
456
457     def test_regexes_are_ignored(self):
458         self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0)
459
460     def test_none_is_invalid(self):
461         self.match(['none'], num_errors=1)
462
463
464 class TestExpectationParserTests(unittest.TestCase):
465     def test_blank(self):
466         (expectation, errors) = TestExpectationParser.parse('')
467         self.assertEqual(expectation.malformed, False)
468         self.assertEqual(expectation.valid, True)
469         self.assertEqual(expectation.comment, None)
470         self.assertEqual(len(errors), 0)
471
472     def test_missing_colon(self):
473         (expectation, errors) = TestExpectationParser.parse('Qux.')
474         self.assertEqual(expectation.malformed, True)
475         self.assertEqual(expectation.valid, False)
476         self.assertEqual(expectation.comment, 'Qux.')
477         self.assertEqual(str(errors), '[("Missing a \':\' in", "\'Qux.\'")]')
478
479     def test_extra_colon(self):
480         (expectation, errors) = TestExpectationParser.parse('FOO : : bar')
481         self.assertEqual(expectation.malformed, True)
482         self.assertEqual(expectation.valid, False)
483         self.assertEqual(expectation.comment, 'FOO : : bar')
484         self.assertEqual(str(errors), '[("Extraneous \':\' in", "\'FOO : : bar\'")]')
485
486     def test_empty_comment(self):
487         (expectation, errors) = TestExpectationParser.parse('//')
488         self.assertEqual(expectation.malformed, False)
489         self.assertEqual(expectation.valid, True)
490         self.assertEqual(expectation.comment, '')
491         self.assertEqual(len(errors), 0)
492
493     def test_comment(self):
494         (expectation, errors) = TestExpectationParser.parse('//Qux.')
495         self.assertEqual(expectation.malformed, False)
496         self.assertEqual(expectation.valid, True)
497         self.assertEqual(expectation.comment, 'Qux.')
498         self.assertEqual(len(errors), 0)
499
500     def test_missing_equal(self):
501         (expectation, errors) = TestExpectationParser.parse('FOO : bar')
502         self.assertEqual(expectation.malformed, True)
503         self.assertEqual(expectation.valid, False)
504         self.assertEqual(expectation.comment, 'FOO : bar')
505         self.assertEqual(str(errors), '[(\'Missing expectations in\', "\'FOO : bar\'")]')
506
507     def test_extra_equal(self):
508         (expectation, errors) = TestExpectationParser.parse('FOO : bar = BAZ = Qux.')
509         self.assertEqual(expectation.malformed, True)
510         self.assertEqual(expectation.valid, False)
511         self.assertEqual(expectation.comment, 'FOO : bar = BAZ = Qux.')
512         self.assertEqual(str(errors), '[("Extraneous \'=\' in", "\'FOO : bar = BAZ = Qux.\'")]')
513
514     def test_valid(self):
515         (expectation, errors) = TestExpectationParser.parse('FOO : bar = BAZ')
516         self.assertEqual(expectation.malformed, False)
517         self.assertEqual(expectation.valid, True)
518         self.assertEqual(expectation.comment, None)
519         self.assertEqual(len(errors), 0)
520
521     def test_valid_with_comment(self):
522         (expectation, errors) = TestExpectationParser.parse('FOO : bar = BAZ //Qux.')
523         self.assertEqual(expectation.malformed, False)
524         self.assertEqual(expectation.valid, True)
525         self.assertEqual(expectation.comment, 'Qux.')
526         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
527         self.assertEqual(str(expectation.expectations), '[\'baz\']')
528         self.assertEqual(len(errors), 0)
529
530     def test_valid_with_multiple_modifiers(self):
531         (expectation, errors) = TestExpectationParser.parse('FOO1 FOO2 : bar = BAZ //Qux.')
532         self.assertEqual(expectation.malformed, False)
533         self.assertEqual(expectation.valid, True)
534         self.assertEqual(expectation.comment, 'Qux.')
535         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
536         self.assertEqual(str(expectation.expectations), '[\'baz\']')
537         self.assertEqual(len(errors), 0)
538
539     def test_line_number_increment(self):
540         validator = TestValidator()
541         expectations = TestExpectationParser.parse_list('// Bar\nFOO : bar = BAZ\n\n', validator)
542         self.assertEqual(str(validator.line_numbers), '[1, 2, 3, 4]')
543
544     def test_validator_feedback(self):
545         validator = TestValidator()
546         expectations = TestExpectationParser.parse_list('FOO : bar1 = BAZ\nFOO : bar2 = BAZ\nFOO : bar3 = BAZ', validator)
547         line_number = 0
548         for expectation in expectations:
549             line_number += 1
550             self.assertEqual(line_number % 2 == 0, expectation.valid)
551
552
553 class TestExpectationSerializerTests(unittest.TestCase):
554     def assert_round_trip(self, in_string, expected_string=None):
555         (expectation, _) = TestExpectationParser.parse(in_string)
556         if expected_string is None:
557             expected_string = in_string
558         self.assertEqual(expected_string, TestExpectationSerializer.to_string(expectation))
559
560     def assert_to_string(self, expectation, expected_string):
561         self.assertEqual(TestExpectationSerializer.to_string(expectation), expected_string)
562
563     def test_string_serializer(self):
564         expectation = TestExpectationLine()
565         self.assert_to_string(expectation, '')
566         expectation.comment = 'Qux.'
567         self.assert_to_string(expectation, '//Qux.')
568         expectation.name = 'bar'
569         self.assert_to_string(expectation, ' : bar =  //Qux.')
570         expectation.modifiers = ['foo']
571         self.assert_to_string(expectation, 'FOO : bar =  //Qux.')
572         expectation.expectations = ['bAz']
573         self.assert_to_string(expectation, 'FOO : bar = BAZ //Qux.')
574         expectation.expectations = ['bAz1', 'baZ2']
575         self.assert_to_string(expectation, 'FOO : bar = BAZ1 BAZ2 //Qux.')
576         expectation.modifiers = ['foo1', 'foO2']
577         self.assert_to_string(expectation, 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
578         expectation.malformed = True
579         self.assert_to_string(expectation, 'Qux.')
580
581     def test_string_roundtrip(self):
582         self.assert_round_trip('')
583         self.assert_round_trip('FOO')
584         self.assert_round_trip(':')
585         self.assert_round_trip('FOO :')
586         self.assert_round_trip('FOO : bar')
587         self.assert_round_trip('  FOO :')
588         self.assert_round_trip('    FOO : bar')
589         self.assert_round_trip('FOO : bar = BAZ')
590         self.assert_round_trip('FOO : bar = BAZ //Qux.')
591         self.assert_round_trip('FOO : bar = BAZ // Qux.')
592         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
593         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
594         self.assert_round_trip('FOO : : bar = BAZ')
595         self.assert_round_trip('FOO : : bar = BAZ')
596         self.assert_round_trip('FOO : : bar ==== BAZ')
597         self.assert_round_trip('=')
598         self.assert_round_trip('//')
599         self.assert_round_trip('// ')
600         self.assert_round_trip('// Foo')
601         self.assert_round_trip('// Foo')
602         self.assert_round_trip('// Foo :')
603         self.assert_round_trip('// Foo : =')
604
605     def test_string_whitespace_stripping(self):
606         self.assert_round_trip('\n', '')
607         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
608         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
609         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
610         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
611         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
612         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
613
614
615 class TestValidator:
616     def __init__(self):
617         self.line_numbers = []
618
619     def validate(self, line_number, expectation, errors):
620         self.line_numbers.append(line_number)
621         return line_number % 2 == 0
622
623 if __name__ == '__main__':
624     unittest.main()