webkitpy: update callers to use port.expectation_dict() instead of test_expectations...
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38 try:
39     from collections import OrderedDict
40 except ImportError:
41     # Needed for Python < 2.7
42     from webkitpy.thirdparty.ordered_dict import OrderedDict
43
44
45 class MockBugManager(object):
46     def close_bug(self, bug_id, reference_bug_id=None):
47         pass
48
49     def create_bug(self):
50         return "BUG_NEWLY_CREATED"
51
52
53 class FunctionsTest(unittest.TestCase):
54     def test_result_was_expected(self):
55         # test basics
56         self.assertEquals(result_was_expected(PASS, set([PASS]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(TEXT, set([PASS]),
59                                               False, False), False)
60
61         # test handling of FAIL expectations
62         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
63                                               False, False), True)
64         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
65                                               False, False), True)
66         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
67                                               False, False), True)
68         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
69                                               False, False), False)
70
71         # test handling of SKIPped tests and results
72         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
73                                               False, True), True)
74         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
75                                               False, False), False)
76
77         # test handling of MISSING results and the REBASELINE modifier
78         self.assertEquals(result_was_expected(MISSING, set([PASS]),
79                                               True, False), True)
80         self.assertEquals(result_was_expected(MISSING, set([PASS]),
81                                               False, False), False)
82
83     def test_remove_pixel_failures(self):
84         self.assertEquals(remove_pixel_failures(set([TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS])),
87                           set([PASS]))
88         self.assertEquals(remove_pixel_failures(set([IMAGE])),
89                           set([PASS]))
90         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
91                           set([TEXT]))
92         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
93                           set([PASS, CRASH]))
94
95     def test_suffixes_for_expectations(self):
96         self.assertEquals(suffixes_for_expectations(set([TEXT])), set(['txt']))
97         self.assertEquals(suffixes_for_expectations(set([FAIL])), set(['txt', 'png']))
98         self.assertEquals(suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png']))
99         self.assertEquals(suffixes_for_expectations(set([IMAGE])), set(['png']))
100         self.assertEquals(suffixes_for_expectations(set([AUDIO])), set(['wav']))
101         self.assertEquals(suffixes_for_expectations(set([TEXT, FAIL, CRASH])), set(['txt', 'png']))
102         self.assertEquals(suffixes_for_expectations(set()), set())
103
104
105 class Base(unittest.TestCase):
106     # Note that all of these tests are written assuming the configuration
107     # being tested is Windows XP, Release build.
108
109     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
110         host = MockHost()
111         self._port = host.port_factory.get('test-win-xp', None)
112         self._exp = None
113         unittest.TestCase.__init__(self, testFunc)
114
115     def get_test(self, test_name):
116         # FIXME: Remove this routine and just reference test names directly.
117         return test_name
118
119     def get_basic_tests(self):
120         return [self.get_test('failures/expected/text.html'),
121                 self.get_test('failures/expected/image_checksum.html'),
122                 self.get_test('failures/expected/crash.html'),
123                 self.get_test('failures/expected/missing_text.html'),
124                 self.get_test('failures/expected/image.html'),
125                 self.get_test('passes/text.html')]
126
127     def get_basic_expectations(self):
128         return """
129 BUG_TEST : failures/expected/text.html = TEXT
130 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
131 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
132 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
133 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
134 """
135
136     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
137         self._expectations_dict = OrderedDict()
138         self._expectations_dict['expectations'] = expectations
139         if overrides:
140             self._expectations_dict['overrides'] = overrides
141         self._port.expectations_dict = lambda: self._expectations_dict
142         self._exp = TestExpectations(self._port, self.get_basic_tests(), is_lint_mode)
143
144     def assert_exp(self, test, result):
145         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
146                           set([result]))
147
148     def assert_bad_expectations(self, expectations, overrides=None):
149         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
150
151
152 class BasicTests(Base):
153     def test_basic(self):
154         self.parse_exp(self.get_basic_expectations())
155         self.assert_exp('failures/expected/text.html', TEXT)
156         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
157         self.assert_exp('passes/text.html', PASS)
158         self.assert_exp('failures/expected/image.html', PASS)
159
160
161 class MiscTests(Base):
162     def test_multiple_results(self):
163         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
164         self.assertEqual(self._exp.get_expectations(
165             self.get_test('failures/expected/text.html')),
166             set([TEXT, CRASH]))
167
168     def test_category_expectations(self):
169         # This test checks unknown tests are not present in the
170         # expectations and that known test part of a test category is
171         # present in the expectations.
172         exp_str = """
173 BUGX WONTFIX : failures/expected = IMAGE
174 """
175         self.parse_exp(exp_str)
176         test_name = 'failures/expected/unknown-test.html'
177         unknown_test = self.get_test(test_name)
178         self.assertRaises(KeyError, self._exp.get_expectations,
179                           unknown_test)
180         self.assert_exp('failures/expected/crash.html', IMAGE)
181
182     def test_get_modifiers(self):
183         self.parse_exp(self.get_basic_expectations())
184         self.assertEqual(self._exp.get_modifiers(
185                          self.get_test('passes/text.html')), [])
186
187     def test_get_expectations_string(self):
188         self.parse_exp(self.get_basic_expectations())
189         self.assertEquals(self._exp.get_expectations_string(
190                           self.get_test('failures/expected/text.html')),
191                           'TEXT')
192
193     def test_expectation_to_string(self):
194         # Normal cases are handled by other tests.
195         self.parse_exp(self.get_basic_expectations())
196         self.assertRaises(ValueError, self._exp.expectation_to_string,
197                           -1)
198
199     def test_get_test_set(self):
200         # Handle some corner cases for this routine not covered by other tests.
201         self.parse_exp(self.get_basic_expectations())
202         s = self._exp.get_test_set(WONTFIX)
203         self.assertEqual(s,
204             set([self.get_test('failures/expected/crash.html'),
205                  self.get_test('failures/expected/image_checksum.html')]))
206         s = self._exp.get_test_set(WONTFIX, CRASH)
207         self.assertEqual(s,
208             set([self.get_test('failures/expected/crash.html')]))
209         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
210         self.assertEqual(s, set([]))
211
212     def test_parse_warning(self):
213         try:
214             self.parse_exp("""FOO : failures/expected/text.html = TEXT
215 SKIP : failures/expected/image.html""", is_lint_mode=True)
216             self.assertFalse(True, "ParseError wasn't raised")
217         except ParseError, e:
218             warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n"
219                         "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
220                         "expectations:2 Missing expectations SKIP : failures/expected/image.html")
221             self.assertEqual(str(e), warnings)
222
223         try:
224             self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
225             self.assertFalse(True, "ParseError wasn't raised")
226         except ParseError, e:
227             warnings = u'expectations:1 Test lacks BUG modifier. failures/expected/text.html'
228             self.assertEqual(str(e), warnings)
229
230     def test_error_on_different_platform(self):
231         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
232         self.assertRaises(ParseError, self.parse_exp,
233             'BUG_TEST MAC : failures/expected/text.html = TEXT\nBUG_TEST MAC : failures/expected/text.html = TEXT',
234             is_lint_mode=True)
235
236     def test_error_on_different_build_type(self):
237         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
238         self.assertRaises(ParseError, self.parse_exp,
239             'BUG_TEST DEBUG : failures/expected/text.html = TEXT\nBUG_TEST DEBUG : failures/expected/text.html = TEXT',
240             is_lint_mode=True)
241
242     def test_overrides(self):
243         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
244                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
245         self.assert_exp('failures/expected/text.html', IMAGE)
246
247     def test_overrides__duplicate(self):
248         self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
249                                      "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
250                                      "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
251
252     def test_pixel_tests_flag(self):
253         def match(test, result, pixel_tests_enabled):
254             return self._exp.matches_an_expected_result(
255                 self.get_test(test), result, pixel_tests_enabled)
256
257         self.parse_exp(self.get_basic_expectations())
258         self.assertTrue(match('failures/expected/text.html', TEXT, True))
259         self.assertTrue(match('failures/expected/text.html', TEXT, False))
260         self.assertFalse(match('failures/expected/text.html', CRASH, True))
261         self.assertFalse(match('failures/expected/text.html', CRASH, False))
262         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
263                               True))
264         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
265                               False))
266         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
267         self.assertTrue(match('passes/text.html', PASS, False))
268
269     def test_more_specific_override_resets_skip(self):
270         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
271                        "BUGX : failures/expected/text.html = IMAGE\n")
272         self.assert_exp('failures/expected/text.html', IMAGE)
273         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
274                                                      'failures/expected/text.html') in
275                          self._exp.get_tests_with_result_type(SKIP))
276
277
278 class SkippedTests(Base):
279     def check(self, expectations, overrides, skips, lint=False):
280         port = MockHost().port_factory.get('qt')
281         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
282         self._expectations_dict = OrderedDict()
283         self._expectations_dict['expectations'] = expectations
284         if overrides:
285             self._expectations_dict['overrides'] = overrides
286         port.expectations_dict = lambda: self._expectations_dict
287         port.skipped_layout_tests = lambda tests: set(skips)
288         exp = TestExpectations(port, ['failures/expected/text.html'], lint)
289
290         # Check that the expectation is for BUG_DUMMY SKIP : ... = PASS
291         self.assertEquals(exp.get_modifiers('failures/expected/text.html'),
292                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER])
293         self.assertEquals(exp.get_expectations('failures/expected/text.html'), set([PASS]))
294
295     def test_skipped_tests_work(self):
296         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
297
298     def test_duplicate_skipped_test_fails_lint(self):
299         self.assertRaises(ParseError, self.check, expectations='BUGX : failures/expected/text.html = text\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
300
301     def test_skipped_file_overrides_expectations(self):
302         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
303                    skips=['failures/expected/text.html'])
304
305     def test_skipped_dir_overrides_expectations(self):
306         self.check(expectations='BUGX : failures/expected/text.html = TEXT\n', overrides=None,
307                    skips=['failures/expected'])
308
309     def test_skipped_file_overrides_overrides(self):
310         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
311                    skips=['failures/expected/text.html'])
312
313     def test_skipped_dir_overrides_overrides(self):
314         self.check(expectations='', overrides='BUGX : failures/expected/text.html = TEXT\n',
315                    skips=['failures/expected'])
316
317
318 class ExpectationSyntaxTests(Base):
319     def test_missing_expectation(self):
320         # This is missing the expectation.
321         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
322
323     def test_missing_colon(self):
324         # This is missing the modifiers and the ':'
325         self.assert_bad_expectations('failures/expected/text.html = TEXT')
326
327     def test_too_many_colons(self):
328         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
329
330     def test_too_many_equals_signs(self):
331         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
332
333     def test_unrecognized_expectation(self):
334         self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
335
336     def test_macro(self):
337         exp_str = """
338 BUG_TEST WIN : failures/expected/text.html = TEXT
339 """
340         self.parse_exp(exp_str)
341         self.assert_exp('failures/expected/text.html', TEXT)
342
343
344 class SemanticTests(Base):
345     def test_bug_format(self):
346         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
347
348     def test_bad_bugid(self):
349         try:
350             self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
351             self.fail('should have raised an error about a bad bug identifier')
352         except ParseError, exp:
353             self.assertEquals(len(exp.warnings), 1)
354
355     def test_missing_bugid(self):
356         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
357         self.assertTrue(self._exp.has_warnings())
358
359     def test_slow_and_timeout(self):
360         # A test cannot be SLOW and expected to TIMEOUT.
361         self.assertRaises(ParseError, self.parse_exp,
362             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
363
364     def test_rebaseline(self):
365         # Can't lint a file w/ 'REBASELINE' in it.
366         self.assertRaises(ParseError, self.parse_exp,
367             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
368             is_lint_mode=True)
369
370     def test_duplicates(self):
371         self.assertRaises(ParseError, self.parse_exp, """
372 BUG_EXP : failures/expected/text.html = TEXT
373 BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
374
375         self.assertRaises(ParseError, self.parse_exp,
376             self.get_basic_expectations(), overrides="""
377 BUG_OVERRIDE : failures/expected/text.html = TEXT
378 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
379
380     def test_missing_file(self):
381         # This should log a non-fatal error.
382         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
383         self.assertTrue(self._exp.has_warnings(), 1)
384
385
386 class PrecedenceTests(Base):
387     def test_file_over_directory(self):
388         # This tests handling precedence of specific lines over directories
389         # and tests expectations covering entire directories.
390         exp_str = """
391 BUGX : failures/expected/text.html = TEXT
392 BUGX WONTFIX : failures/expected = IMAGE
393 """
394         self.parse_exp(exp_str)
395         self.assert_exp('failures/expected/text.html', TEXT)
396         self.assert_exp('failures/expected/crash.html', IMAGE)
397
398         exp_str = """
399 BUGX WONTFIX : failures/expected = IMAGE
400 BUGX : failures/expected/text.html = TEXT
401 """
402         self.parse_exp(exp_str)
403         self.assert_exp('failures/expected/text.html', TEXT)
404         self.assert_exp('failures/expected/crash.html', IMAGE)
405
406     def test_ambiguous(self):
407         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
408                                      "BUG_TEST WIN : passes/text.html = FAIL\n")
409
410     def test_more_modifiers(self):
411         self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
412                                      "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
413
414     def test_order_in_file(self):
415         self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
416                                      "BUG_TEST RELEASE : passes/text.html = PASS\n")
417
418     def test_macro_overrides(self):
419         self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
420                                      "BUG_TEST XP : passes/text.html = TEXT\n")
421
422
423 class RemoveConfigurationsTest(Base):
424     def test_remove(self):
425         host = MockHost()
426         test_port = host.port_factory.get('test-win-xp', None)
427         test_port.test_exists = lambda test: True
428         test_port.test_isfile = lambda test: True
429
430         test_config = test_port.test_configuration()
431         test_port.expectations_dict = lambda: {"expectations": """BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT
432 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
433 """}
434         expectations = TestExpectations(test_port, self.get_basic_tests())
435
436         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
437
438         self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT
439 BUGY WIN MAC DEBUG : failures/expected/foo.html = CRASH
440 """, actual_expectations)
441
442     def test_remove_line(self):
443         host = MockHost()
444         test_port = host.port_factory.get('test-win-xp', None)
445         test_port.test_exists = lambda test: True
446         test_port.test_isfile = lambda test: True
447
448         test_config = test_port.test_configuration()
449         test_port.expectations_dict = lambda: {'expectations': """BUGX WIN RELEASE : failures/expected/foo.html = TEXT
450 BUGY WIN DEBUG : failures/expected/foo.html = CRASH
451 """}
452         expectations = TestExpectations(test_port)
453
454         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
455         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
456         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
457
458         self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH
459 """, actual_expectations)
460
461
462 class RebaseliningTest(Base):
463     """Test rebaselining-specific functionality."""
464     def assertRemove(self, input_expectations, tests, expected_expectations):
465         self.parse_exp(input_expectations, is_lint_mode=False)
466         actual_expectations = self._exp.remove_rebaselined_tests(tests)
467         self.assertEqual(expected_expectations, actual_expectations)
468
469     def test_remove(self):
470         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
471                           'BUGY : failures/expected/image.html = IMAGE\n'
472                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
473                           ['failures/expected/text.html'],
474                           'BUGY : failures/expected/image.html = IMAGE\n'
475                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
476
477     def test_no_get_rebaselining_failures(self):
478         self.parse_exp(self.get_basic_expectations())
479         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
480
481
482 class TestExpectationParserTests(unittest.TestCase):
483     def _tokenize(self, line):
484         return TestExpectationParser._tokenize('path', line, 0)
485
486     def test_tokenize_blank(self):
487         expectation = self._tokenize('')
488         self.assertEqual(expectation.comment, None)
489         self.assertEqual(len(expectation.warnings), 0)
490
491     def test_tokenize_missing_colon(self):
492         expectation = self._tokenize('Qux.')
493         self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
494
495     def test_tokenize_extra_colon(self):
496         expectation = self._tokenize('FOO : : bar')
497         self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
498
499     def test_tokenize_empty_comment(self):
500         expectation = self._tokenize('//')
501         self.assertEqual(expectation.comment, '')
502         self.assertEqual(len(expectation.warnings), 0)
503
504     def test_tokenize_comment(self):
505         expectation = self._tokenize('//Qux.')
506         self.assertEqual(expectation.comment, 'Qux.')
507         self.assertEqual(len(expectation.warnings), 0)
508
509     def test_tokenize_missing_equal(self):
510         expectation = self._tokenize('FOO : bar')
511         self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
512
513     def test_tokenize_extra_equal(self):
514         expectation = self._tokenize('FOO : bar = BAZ = Qux.')
515         self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
516
517     def test_tokenize_valid(self):
518         expectation = self._tokenize('FOO : bar = BAZ')
519         self.assertEqual(expectation.comment, None)
520         self.assertEqual(len(expectation.warnings), 0)
521
522     def test_tokenize_valid_with_comment(self):
523         expectation = self._tokenize('FOO : bar = BAZ //Qux.')
524         self.assertEqual(expectation.comment, 'Qux.')
525         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
526         self.assertEqual(str(expectation.expectations), '[\'baz\']')
527         self.assertEqual(len(expectation.warnings), 0)
528
529     def test_tokenize_valid_with_multiple_modifiers(self):
530         expectation = self._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
531         self.assertEqual(expectation.comment, 'Qux.')
532         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
533         self.assertEqual(str(expectation.expectations), '[\'baz\']')
534         self.assertEqual(len(expectation.warnings), 0)
535
536     def test_parse_empty_string(self):
537         host = MockHost()
538         test_port = host.port_factory.get('test-win-xp', None)
539         test_port.test_exists = lambda test: True
540         test_config = test_port.test_configuration()
541         full_test_list = []
542         expectation_line = self._tokenize('')
543         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
544         parser._parse_line(expectation_line)
545         self.assertFalse(expectation_line.is_invalid())
546
547
548 class TestExpectationSerializerTests(unittest.TestCase):
549     def __init__(self, testFunc):
550         host = MockHost()
551         test_port = host.port_factory.get('test-win-xp', None)
552         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
553         self._serializer = TestExpectationSerializer(self._converter)
554         unittest.TestCase.__init__(self, testFunc)
555
556     def _tokenize(self, line):
557         return TestExpectationParser._tokenize('path', line, 0)
558
559     def assert_round_trip(self, in_string, expected_string=None):
560         expectation = self._tokenize(in_string)
561         if expected_string is None:
562             expected_string = in_string
563         self.assertEqual(expected_string, self._serializer.to_string(expectation))
564
565     def assert_list_round_trip(self, in_string, expected_string=None):
566         expectations = TestExpectationParser._tokenize_list('path', in_string)
567         if expected_string is None:
568             expected_string = in_string
569         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, self._converter))
570
571     def test_unparsed_to_string(self):
572         expectation = TestExpectationLine()
573         serializer = TestExpectationSerializer()
574
575         self.assertEqual(serializer.to_string(expectation), '')
576         expectation.comment = 'Qux.'
577         self.assertEqual(serializer.to_string(expectation), '//Qux.')
578         expectation.name = 'bar'
579         self.assertEqual(serializer.to_string(expectation), ' : bar =  //Qux.')
580         expectation.modifiers = ['foo']
581         self.assertEqual(serializer.to_string(expectation), 'FOO : bar =  //Qux.')
582         expectation.expectations = ['bAz']
583         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
584         expectation.expectations = ['bAz1', 'baZ2']
585         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
586         expectation.modifiers = ['foo1', 'foO2']
587         self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
588         expectation.warnings.append('Oh the horror.')
589         self.assertEqual(serializer.to_string(expectation), '')
590         expectation.original_string = 'Yes it is!'
591         self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
592
593     def test_unparsed_list_to_string(self):
594         expectation = TestExpectationLine()
595         expectation.comment = 'Qux.'
596         expectation.name = 'bar'
597         expectation.modifiers = ['foo']
598         expectation.expectations = ['bAz1', 'baZ2']
599         self.assertEqual(TestExpectationSerializer.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
600
601     def test_parsed_to_string(self):
602         expectation_line = TestExpectationLine()
603         expectation_line.parsed_bug_modifiers = ['BUGX']
604         expectation_line.name = 'test/name/for/realz.html'
605         expectation_line.parsed_expectations = set([IMAGE])
606         self.assertEqual(self._serializer.to_string(expectation_line), None)
607         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
608         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
609         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
610         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP : test/name/for/realz.html = IMAGE')
611
612     def test_parsed_expectations_string(self):
613         expectation_line = TestExpectationLine()
614         expectation_line.parsed_expectations = set([])
615         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
616         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
617         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
618         expectation_line.parsed_expectations = set([PASS, FAIL])
619         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
620         expectation_line.parsed_expectations = set([FAIL, PASS])
621         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
622
623     def test_parsed_modifier_string(self):
624         expectation_line = TestExpectationLine()
625         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
626         expectation_line.parsed_modifiers = ['for', 'the']
627         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
628         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
629         expectation_line.parsed_bug_modifiers = []
630         expectation_line.parsed_modifiers = []
631         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
632         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
633         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
634         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
635         expectation_line.parsed_bug_modifiers = []
636         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
637         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
638
639     def test_format_result(self):
640         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
641         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
642
643     def test_string_roundtrip(self):
644         self.assert_round_trip('')
645         self.assert_round_trip('FOO')
646         self.assert_round_trip(':')
647         self.assert_round_trip('FOO :')
648         self.assert_round_trip('FOO : bar')
649         self.assert_round_trip('  FOO :')
650         self.assert_round_trip('    FOO : bar')
651         self.assert_round_trip('FOO : bar = BAZ')
652         self.assert_round_trip('FOO : bar = BAZ //Qux.')
653         self.assert_round_trip('FOO : bar = BAZ // Qux.')
654         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
655         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
656         self.assert_round_trip('FOO : : bar = BAZ')
657         self.assert_round_trip('FOO : : bar = BAZ')
658         self.assert_round_trip('FOO : : bar ==== BAZ')
659         self.assert_round_trip('=')
660         self.assert_round_trip('//')
661         self.assert_round_trip('// ')
662         self.assert_round_trip('// Foo')
663         self.assert_round_trip('// Foo')
664         self.assert_round_trip('// Foo :')
665         self.assert_round_trip('// Foo : =')
666
667     def test_list_roundtrip(self):
668         self.assert_list_round_trip('')
669         self.assert_list_round_trip('\n')
670         self.assert_list_round_trip('\n\n')
671         self.assert_list_round_trip('bar')
672         self.assert_list_round_trip('bar\n//Qux.')
673         self.assert_list_round_trip('bar\n//Qux.\n')
674
675     def test_reconstitute_only_these(self):
676         lines = []
677         reconstitute_only_these = []
678
679         def add_line(matching_configurations, reconstitute):
680             expectation_line = TestExpectationLine()
681             expectation_line.original_string = "Nay"
682             expectation_line.parsed_bug_modifiers = ['BUGX']
683             expectation_line.name = 'Yay'
684             expectation_line.parsed_expectations = set([IMAGE])
685             expectation_line.matching_configurations = matching_configurations
686             lines.append(expectation_line)
687             if reconstitute:
688                 reconstitute_only_these.append(expectation_line)
689
690         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
691         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
692         serialized = TestExpectationSerializer.list_to_string(lines, self._converter)
693         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nBUGX XP : Yay = IMAGE")
694         serialized = TestExpectationSerializer.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
695         self.assertEquals(serialized, "BUGX XP RELEASE : Yay = IMAGE\nNay")
696
697     def test_string_whitespace_stripping(self):
698         self.assert_round_trip('\n', '')
699         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
700         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
701         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
702         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
703         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
704         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
705
706
707 if __name__ == '__main__':
708     unittest.main()