nrwt: implement the actual cascade of TestExpectations
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38 try:
39     from collections import OrderedDict
40 except ImportError:
41     # Needed for Python < 2.7
42     from webkitpy.thirdparty.ordered_dict import OrderedDict
43
44
45 class MockBugManager(object):
46     def close_bug(self, bug_id, reference_bug_id=None):
47         pass
48
49     def create_bug(self):
50         return "BUG_NEWLY_CREATED"
51
52
53 class FunctionsTest(unittest.TestCase):
54     def test_result_was_expected(self):
55         # test basics
56         self.assertEquals(result_was_expected(PASS, set([PASS]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(TEXT, set([PASS]),
59                                               False, False), False)
60
61         # test handling of FAIL expectations
62         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
63                                               False, False), True)
64         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
65                                               False, False), True)
66         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
67                                               False, False), True)
68         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
69                                               False, False), False)
70
71         # test handling of SKIPped tests and results
72         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
73                                               False, True), True)
74         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
75                                               False, False), False)
76
77         # test handling of MISSING results and the REBASELINE modifier
78         self.assertEquals(result_was_expected(MISSING, set([PASS]),
79                                               True, False), True)
80         self.assertEquals(result_was_expected(MISSING, set([PASS]),
81                                               False, False), False)
82
83     def test_remove_pixel_failures(self):
84         self.assertEquals(remove_pixel_failures(set([TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS])),
87                           set([PASS]))
88         self.assertEquals(remove_pixel_failures(set([IMAGE])),
89                           set([PASS]))
90         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
91                           set([TEXT]))
92         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
93                           set([PASS, CRASH]))
94
95     def test_suffixes_for_expectations(self):
96         self.assertEquals(suffixes_for_expectations(set([TEXT])), set(['txt']))
97         self.assertEquals(suffixes_for_expectations(set([FAIL])), set(['txt', 'png']))
98         self.assertEquals(suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png']))
99         self.assertEquals(suffixes_for_expectations(set([IMAGE])), set(['png']))
100         self.assertEquals(suffixes_for_expectations(set([AUDIO])), set(['wav']))
101         self.assertEquals(suffixes_for_expectations(set([TEXT, FAIL, CRASH])), set(['txt', 'png']))
102         self.assertEquals(suffixes_for_expectations(set()), set())
103
104
105 class Base(unittest.TestCase):
106     # Note that all of these tests are written assuming the configuration
107     # being tested is Windows XP, Release build.
108
109     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
110         host = MockHost()
111         self._port = host.port_factory.get('test-win-xp', None)
112         self._exp = None
113         unittest.TestCase.__init__(self, testFunc)
114
115     def get_test(self, test_name):
116         # FIXME: Remove this routine and just reference test names directly.
117         return test_name
118
119     def get_basic_tests(self):
120         return [self.get_test('failures/expected/text.html'),
121                 self.get_test('failures/expected/image_checksum.html'),
122                 self.get_test('failures/expected/crash.html'),
123                 self.get_test('failures/expected/missing_text.html'),
124                 self.get_test('failures/expected/image.html'),
125                 self.get_test('passes/text.html')]
126
127     def get_basic_expectations(self):
128         return """
129 BUG_TEST : failures/expected/text.html = TEXT
130 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
131 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
132 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
133 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
134 """
135
136     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
137         self._expectations_dict = OrderedDict()
138         self._expectations_dict['expectations'] = expectations
139         if overrides:
140             self._expectations_dict['overrides'] = overrides
141         self._port.expectations_dict = lambda: self._expectations_dict
142         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
143
144     def assert_exp(self, test, result):
145         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
146                           set([result]))
147
148     def assert_bad_expectations(self, expectations, overrides=None):
149         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
150
151
152 class BasicTests(Base):
153     def test_basic(self):
154         self.parse_exp(self.get_basic_expectations())
155         self.assert_exp('failures/expected/text.html', TEXT)
156         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
157         self.assert_exp('passes/text.html', PASS)
158         self.assert_exp('failures/expected/image.html', PASS)
159
160
161 class MiscTests(Base):
162     def test_multiple_results(self):
163         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
164         self.assertEqual(self._exp.get_expectations(
165             self.get_test('failures/expected/text.html')),
166             set([TEXT, CRASH]))
167
168     def test_category_expectations(self):
169         # This test checks unknown tests are not present in the
170         # expectations and that known test part of a test category is
171         # present in the expectations.
172         exp_str = """
173 BUGX WONTFIX : failures/expected = IMAGE
174 """
175         self.parse_exp(exp_str)
176         test_name = 'failures/expected/unknown-test.html'
177         unknown_test = self.get_test(test_name)
178         self.assertRaises(KeyError, self._exp.get_expectations,
179                           unknown_test)
180         self.assert_exp('failures/expected/crash.html', IMAGE)
181
182     def test_get_modifiers(self):
183         self.parse_exp(self.get_basic_expectations())
184         self.assertEqual(self._exp.get_modifiers(
185                          self.get_test('passes/text.html')), [])
186
187     def test_get_expectations_string(self):
188         self.parse_exp(self.get_basic_expectations())
189         self.assertEquals(self._exp.get_expectations_string(
190                           self.get_test('failures/expected/text.html')),
191                           'TEXT')
192
193     def test_expectation_to_string(self):
194         # Normal cases are handled by other tests.
195         self.parse_exp(self.get_basic_expectations())
196         self.assertRaises(ValueError, self._exp.expectation_to_string,
197                           -1)
198
199     def test_get_test_set(self):
200         # Handle some corner cases for this routine not covered by other tests.
201         self.parse_exp(self.get_basic_expectations())
202         s = self._exp.get_test_set(WONTFIX)
203         self.assertEqual(s,
204             set([self.get_test('failures/expected/crash.html'),
205                  self.get_test('failures/expected/image_checksum.html')]))
206         s = self._exp.get_test_set(WONTFIX, CRASH)
207         self.assertEqual(s,
208             set([self.get_test('failures/expected/crash.html')]))
209         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
210         self.assertEqual(s, set([]))
211
212     def test_parse_warning(self):
213         try:
214             self.parse_exp("""FOO : failures/expected/text.html = TEXT
215 SKIP : failures/expected/image.html""", is_lint_mode=True)
216             self.assertFalse(True, "ParseError wasn't raised")
217         except ParseError, e:
218             warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n"
219                         "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
220                         "expectations:2 Missing expectations SKIP : failures/expected/image.html")
221             self.assertEqual(str(e), warnings)
222
223         try:
224             self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
225             self.assertFalse(True, "ParseError wasn't raised")
226         except ParseError, e:
227             warnings = u'expectations:1 Test lacks BUG modifier. failures/expected/text.html'
228             self.assertEqual(str(e), warnings)
229
230     def test_error_on_different_platform(self):
231         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
232         self.assertRaises(ParseError, self.parse_exp,
233             'BUG_TEST MAC : failures/expected/text.html = TEXT\nBUG_TEST MAC : failures/expected/text.html = TEXT',
234             is_lint_mode=True)
235
236     def test_error_on_different_build_type(self):
237         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
238         self.assertRaises(ParseError, self.parse_exp,
239             'BUG_TEST DEBUG : failures/expected/text.html = TEXT\nBUG_TEST DEBUG : failures/expected/text.html = TEXT',
240             is_lint_mode=True)
241
242     def test_overrides(self):
243         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
244                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
245         self.assert_exp('failures/expected/text.html', IMAGE)
246
247     def test_overrides__directory(self):
248         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
249                        "BUG_OVERRIDE: failures/expected = CRASH")
250         self.assert_exp('failures/expected/text.html', CRASH)
251         self.assert_exp('failures/expected/image.html', CRASH)
252
253     def test_overrides__duplicate(self):
254         self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
255                                      "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
256                                      "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
257
258     def test_pixel_tests_flag(self):
259         def match(test, result, pixel_tests_enabled):
260             return self._exp.matches_an_expected_result(
261                 self.get_test(test), result, pixel_tests_enabled)
262
263         self.parse_exp(self.get_basic_expectations())
264         self.assertTrue(match('failures/expected/text.html', TEXT, True))
265         self.assertTrue(match('failures/expected/text.html', TEXT, False))
266         self.assertFalse(match('failures/expected/text.html', CRASH, True))
267         self.assertFalse(match('failures/expected/text.html', CRASH, False))
268         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
269                               True))
270         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
271                               False))
272         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
273         self.assertTrue(match('passes/text.html', PASS, False))
274
275     def test_more_specific_override_resets_skip(self):
276         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
277                        "BUGX : failures/expected/text.html = IMAGE\n")
278         self.assert_exp('failures/expected/text.html', IMAGE)
279         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
280                                                      'failures/expected/text.html') in
281                          self._exp.get_tests_with_result_type(SKIP))
282
283
284 class SkippedTests(Base):
285     def check(self, expectations, overrides, skips, lint=False):
286         port = MockHost().port_factory.get('qt')
287         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
288         self._expectations_dict = OrderedDict()
289         self._expectations_dict['expectations'] = expectations
290         if overrides:
291             self._expectations_dict['overrides'] = overrides
292         port.expectations_dict = lambda: self._expectations_dict
293         port.skipped_layout_tests = lambda tests: set(skips)
294         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
295
296         # Check that the expectation is for BUG_DUMMY SKIP : ... = PASS
297         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
298                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER])
299         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
300
301     def test_skipped_tests_work(self):
302         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
303
304     def test_duplicate_skipped_test_fails_lint(self):
305         self.assertRaises(ParseError, self.check, expectations='BUGX : failures/expected/text.html = text\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
306
307     def test_skipped_file_overrides_expectations(self):
308         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
309                    skips=['failures/expected/text.html'])
310
311     def test_skipped_dir_overrides_expectations(self):
312         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
313                    skips=['failures/expected'])
314
315     def test_skipped_file_overrides_overrides(self):
316         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
317                    skips=['failures/expected/text.html'])
318
319     def test_skipped_dir_overrides_overrides(self):
320         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
321                    skips=['failures/expected'])
322
323
324 class ExpectationSyntaxTests(Base):
325     def test_missing_expectation(self):
326         # This is missing the expectation.
327         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
328
329     def test_missing_colon(self):
330         # This is missing the modifiers and the ':'
331         self.assert_bad_expectations('failures/expected/text.html = TEXT')
332
333     def test_too_many_colons(self):
334         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
335
336     def test_too_many_equals_signs(self):
337         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
338
339     def test_unrecognized_expectation(self):
340         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
341
342     def test_macro(self):
343         exp_str = """
344 BUG_TEST WIN : failures/expected/text.html = TEXT
345 """
346         self.parse_exp(exp_str)
347         self.assert_exp('failures/expected/text.html', TEXT)
348
349
350 class SemanticTests(Base):
351     def test_bug_format(self):
352         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
353
354     def test_bad_bugid(self):
355         try:
356             self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
357             self.fail('should have raised an error about a bad bug identifier')
358         except ParseError, exp:
359             self.assertEquals(len(exp.warnings), 1)
360
361     def test_missing_bugid(self):
362         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
363         self.assertTrue(self._exp.has_warnings())
364
365     def test_slow_and_timeout(self):
366         # A test cannot be SLOW and expected to TIMEOUT.
367         self.assertRaises(ParseError, self.parse_exp,
368             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
369
370     def test_rebaseline(self):
371         # Can't lint a file w/ 'REBASELINE' in it.
372         self.assertRaises(ParseError, self.parse_exp,
373             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
374             is_lint_mode=True)
375
376     def test_duplicates(self):
377         self.assertRaises(ParseError, self.parse_exp, """
378 BUG_EXP : failures/expected/text.html = TEXT
379 BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
380
381         self.assertRaises(ParseError, self.parse_exp,
382             self.get_basic_expectations(), overrides="""
383 BUG_OVERRIDE : failures/expected/text.html = TEXT
384 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
385
386     def test_missing_file(self):
387         # This should log a non-fatal error.
388         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
389         self.assertTrue(self._exp.has_warnings(), 1)
390
391
392 class PrecedenceTests(Base):
393     def test_file_over_directory(self):
394         # This tests handling precedence of specific lines over directories
395         # and tests expectations covering entire directories.
396         exp_str = """
397 BUGX : failures/expected/text.html = TEXT
398 BUGX WONTFIX : failures/expected = IMAGE
399 """
400         self.parse_exp(exp_str)
401         self.assert_exp('failures/expected/text.html', TEXT)
402         self.assert_exp('failures/expected/crash.html', IMAGE)
403
404         exp_str = """
405 BUGX WONTFIX : failures/expected = IMAGE
406 BUGX : failures/expected/text.html = TEXT
407 """
408         self.parse_exp(exp_str)
409         self.assert_exp('failures/expected/text.html', TEXT)
410         self.assert_exp('failures/expected/crash.html', IMAGE)
411
412     def test_ambiguous(self):
413         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
414                                      "BUG_TEST WIN : passes/text.html = FAIL\n")
415
416     def test_more_modifiers(self):
417         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
418                                      "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
419
420     def test_order_in_file(self):
421         self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
422                                      "BUG_TEST RELEASE : passes/text.html = PASS\n")
423
424     def test_macro_overrides(self):
425         self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
426                                      "BUG_TEST XP : passes/text.html = TEXT\n")
427
428
429 class RemoveConfigurationsTest(Base):
430     def test_remove(self):
431         host = MockHost()
432         test_port = host.port_factory.get('test-win-xp', None)
433         test_port.test_exists = lambda test: True
434         test_port.test_isfile = lambda test: True
435
436         test_config = test_port.test_configuration()
437         test_port.expectations_dict = lambda: {"expectations": """BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT
438 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
439 """}
440         expectations = TestExpectations(test_port, self.get_basic_tests())
441
442         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
443
444         self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT
445 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
446 """, actual_expectations)
447
448     def test_remove_line(self):
449         host = MockHost()
450         test_port = host.port_factory.get('test-win-xp', None)
451         test_port.test_exists = lambda test: True
452         test_port.test_isfile = lambda test: True
453
454         test_config = test_port.test_configuration()
455         test_port.expectations_dict = lambda: {'expectations': """BUGX WIN RELEASE : failures/expected/foo.html = TEXT
456 BUGY WIN DEBUG : failures/expected/foo.html = CRASH
457 """}
458         expectations = TestExpectations(test_port)
459
460         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
461         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
462         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
463
464         self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH
465 """, actual_expectations)
466
467
468 class RebaseliningTest(Base):
469     """Test rebaselining-specific functionality."""
470     def assertRemove(self, input_expectations, tests, expected_expectations):
471         self.parse_exp(input_expectations, is_lint_mode=False)
472         actual_expectations = self._exp.remove_rebaselined_tests(tests)
473         self.assertEqual(expected_expectations, actual_expectations)
474
475     def test_remove(self):
476         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
477                           'BUGY : failures/expected/image.html = IMAGE\n'
478                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
479                           ['failures/expected/text.html'],
480                           'BUGY : failures/expected/image.html = IMAGE\n'
481                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
482
483     def test_no_get_rebaselining_failures(self):
484         self.parse_exp(self.get_basic_expectations())
485         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
486
487
488 class TestExpectationParserTests(unittest.TestCase):
489     def _tokenize(self, line):
490         return TestExpectationParser._tokenize('path', line, 0)
491
492     def test_tokenize_blank(self):
493         expectation = self._tokenize('')
494         self.assertEqual(expectation.comment, None)
495         self.assertEqual(len(expectation.warnings), 0)
496
497     def test_tokenize_missing_colon(self):
498         expectation = self._tokenize('Qux.')
499         self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
500
501     def test_tokenize_extra_colon(self):
502         expectation = self._tokenize('FOO : : bar')
503         self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
504
505     def test_tokenize_empty_comment(self):
506         expectation = self._tokenize('//')
507         self.assertEqual(expectation.comment, '')
508         self.assertEqual(len(expectation.warnings), 0)
509
510     def test_tokenize_comment(self):
511         expectation = self._tokenize('//Qux.')
512         self.assertEqual(expectation.comment, 'Qux.')
513         self.assertEqual(len(expectation.warnings), 0)
514
515     def test_tokenize_missing_equal(self):
516         expectation = self._tokenize('FOO : bar')
517         self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
518
519     def test_tokenize_extra_equal(self):
520         expectation = self._tokenize('FOO : bar = BAZ = Qux.')
521         self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
522
523     def test_tokenize_valid(self):
524         expectation = self._tokenize('FOO : bar = BAZ')
525         self.assertEqual(expectation.comment, None)
526         self.assertEqual(len(expectation.warnings), 0)
527
528     def test_tokenize_valid_with_comment(self):
529         expectation = self._tokenize('FOO : bar = BAZ //Qux.')
530         self.assertEqual(expectation.comment, 'Qux.')
531         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
532         self.assertEqual(str(expectation.expectations), '[\'baz\']')
533         self.assertEqual(len(expectation.warnings), 0)
534
535     def test_tokenize_valid_with_multiple_modifiers(self):
536         expectation = self._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
537         self.assertEqual(expectation.comment, 'Qux.')
538         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
539         self.assertEqual(str(expectation.expectations), '[\'baz\']')
540         self.assertEqual(len(expectation.warnings), 0)
541
542     def test_parse_empty_string(self):
543         host = MockHost()
544         test_port = host.port_factory.get('test-win-xp', None)
545         test_port.test_exists = lambda test: True
546         test_config = test_port.test_configuration()
547         full_test_list = []
548         expectation_line = self._tokenize('')
549         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
550         parser._parse_line(expectation_line)
551         self.assertFalse(expectation_line.is_invalid())
552
553
554 class TestExpectationSerializerTests(unittest.TestCase):
555     def __init__(self, testFunc):
556         host = MockHost()
557         test_port = host.port_factory.get('test-win-xp', None)
558         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
559         self._serializer = TestExpectationSerializer(self._converter)
560         unittest.TestCase.__init__(self, testFunc)
561
562     def _tokenize(self, line):
563         return TestExpectationParser._tokenize('path', line, 0)
564
565     def assert_round_trip(self, in_string, expected_string=None):
566         expectation = self._tokenize(in_string)
567         if expected_string is None:
568             expected_string = in_string
569         self.assertEqual(expected_string, self._serializer.to_string(expectation))
570
571     def assert_list_round_trip(self, in_string, expected_string=None):
572         expectations = TestExpectationParser._tokenize_list('path', in_string)
573         if expected_string is None:
574             expected_string = in_string
575         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, self._converter))
576
577     def test_unparsed_to_string(self):
578         expectation = TestExpectationLine()
579         serializer = TestExpectationSerializer()
580
581         self.assertEqual(serializer.to_string(expectation), '')
582         expectation.comment = 'Qux.'
583         self.assertEqual(serializer.to_string(expectation), '//Qux.')
584         expectation.name = 'bar'
585         self.assertEqual(serializer.to_string(expectation), ' : bar =  //Qux.')
586         expectation.modifiers = ['foo']
587         self.assertEqual(serializer.to_string(expectation), 'FOO : bar =  //Qux.')
588         expectation.expectations = ['bAz']
589         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
590         expectation.expectations = ['bAz1', 'baZ2']
591         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
592         expectation.modifiers = ['foo1', 'foO2']
593         self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
594         expectation.warnings.append('Oh the horror.')
595         self.assertEqual(serializer.to_string(expectation), '')
596         expectation.original_string = 'Yes it is!'
597         self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
598
599     def test_unparsed_list_to_string(self):
600         expectation = TestExpectationLine()
601         expectation.comment = 'Qux.'
602         expectation.name = 'bar'
603         expectation.modifiers = ['foo']
604         expectation.expectations = ['bAz1', 'baZ2']
605         self.assertEqual(TestExpectationSerializer.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
606
607     def test_parsed_to_string(self):
608         expectation_line = TestExpectationLine()
609         expectation_line.parsed_bug_modifiers = ['BUGX']
610         expectation_line.name = 'test/name/for/realz.html'
611         expectation_line.parsed_expectations = set([IMAGE])
612         self.assertEqual(self._serializer.to_string(expectation_line), None)
613         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
614         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
615         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
616         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP : test/name/for/realz.html = IMAGE')
617
618     def test_parsed_expectations_string(self):
619         expectation_line = TestExpectationLine()
620         expectation_line.parsed_expectations = set([])
621         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
622         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
623         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
624         expectation_line.parsed_expectations = set([PASS, FAIL])
625         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
626         expectation_line.parsed_expectations = set([FAIL, PASS])
627         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
628
629     def test_parsed_modifier_string(self):
630         expectation_line = TestExpectationLine()
631         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
632         expectation_line.parsed_modifiers = ['for', 'the']
633         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
634         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
635         expectation_line.parsed_bug_modifiers = []
636         expectation_line.parsed_modifiers = []
637         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
638         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
639         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
640         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
641         expectation_line.parsed_bug_modifiers = []
642         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
643         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
644
645     def test_format_result(self):
646         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
647         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
648
649     def test_string_roundtrip(self):
650         self.assert_round_trip('')
651         self.assert_round_trip('FOO')
652         self.assert_round_trip(':')
653         self.assert_round_trip('FOO :')
654         self.assert_round_trip('FOO : bar')
655         self.assert_round_trip('  FOO :')
656         self.assert_round_trip('    FOO : bar')
657         self.assert_round_trip('FOO : bar = BAZ')
658         self.assert_round_trip('FOO : bar = BAZ //Qux.')
659         self.assert_round_trip('FOO : bar = BAZ // Qux.')
660         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
661         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
662         self.assert_round_trip('FOO : : bar = BAZ')
663         self.assert_round_trip('FOO : : bar = BAZ')
664         self.assert_round_trip('FOO : : bar ==== BAZ')
665         self.assert_round_trip('=')
666         self.assert_round_trip('//')
667         self.assert_round_trip('// ')
668         self.assert_round_trip('// Foo')
669         self.assert_round_trip('// Foo')
670         self.assert_round_trip('// Foo :')
671         self.assert_round_trip('// Foo : =')
672
673     def test_list_roundtrip(self):
674         self.assert_list_round_trip('')
675         self.assert_list_round_trip('\n')
676         self.assert_list_round_trip('\n\n')
677         self.assert_list_round_trip('bar')
678         self.assert_list_round_trip('bar\n//Qux.')
679         self.assert_list_round_trip('bar\n//Qux.\n')
680
681     def test_reconstitute_only_these(self):
682         lines = []
683         reconstitute_only_these = []
684
685         def add_line(matching_configurations, reconstitute):
686             expectation_line = TestExpectationLine()
687             expectation_line.original_string = "Nay"
688             expectation_line.parsed_bug_modifiers = ['BUGX']
689             expectation_line.name = 'Yay'
690             expectation_line.parsed_expectations = set([IMAGE])
691             expectation_line.matching_configurations = matching_configurations
692             lines.append(expectation_line)
693             if reconstitute:
694                 reconstitute_only_these.append(expectation_line)
695
696         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
697         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
698         serialized = TestExpectationSerializer.list_to_string(lines, self._converter)
699         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nBUGX XP : Yay = IMAGE")
700         serialized = TestExpectationSerializer.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
701         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nNay")
702
703     def test_string_whitespace_stripping(self):
704         self.assert_round_trip('\n', '')
705         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
706         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
707         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
708         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
709         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
710         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
711
712
713 if __name__ == '__main__':
714     unittest.main()