TestExpectationsEditor should return a list of updated expectations.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.layout_tests import port
33 from webkitpy.layout_tests.port import base
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38
39 class MockBugManager(object):
40     def close_bug(self, bug_id, reference_bug_id=None):
41         pass
42
43     def create_bug(self):
44         return "BUG_NEWLY_CREATED"
45
46
47 class FunctionsTest(unittest.TestCase):
48     def test_result_was_expected(self):
49         # test basics
50         self.assertEquals(result_was_expected(PASS, set([PASS]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([PASS]),
53                                               False, False), False)
54
55         # test handling of FAIL expectations
56         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
59                                               False, False), True)
60         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
61                                               False, False), True)
62         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
63                                               False, False), False)
64
65         # test handling of SKIPped tests and results
66         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
67                                               False, True), True)
68         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
69                                               False, False), False)
70
71         # test handling of MISSING results and the REBASELINE modifier
72         self.assertEquals(result_was_expected(MISSING, set([PASS]),
73                                               True, False), True)
74         self.assertEquals(result_was_expected(MISSING, set([PASS]),
75                                               False, False), False)
76
77     def test_remove_pixel_failures(self):
78         self.assertEquals(remove_pixel_failures(set([TEXT])),
79                           set([TEXT]))
80         self.assertEquals(remove_pixel_failures(set([PASS])),
81                           set([PASS]))
82         self.assertEquals(remove_pixel_failures(set([IMAGE])),
83                           set([PASS]))
84         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
87                           set([PASS, CRASH]))
88
89
90 class Base(unittest.TestCase):
91     # Note that all of these tests are written assuming the configuration
92     # being tested is Windows XP, Release build.
93
94     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
95         self._port = port.get('test-win-xp', None)
96         self._fs = self._port._filesystem
97         self._exp = None
98         unittest.TestCase.__init__(self, testFunc)
99
100     def get_test(self, test_name):
101         # FIXME: Remove this routine and just reference test names directly.
102         return test_name
103
104     def get_basic_tests(self):
105         return [self.get_test('failures/expected/text.html'),
106                 self.get_test('failures/expected/image_checksum.html'),
107                 self.get_test('failures/expected/crash.html'),
108                 self.get_test('failures/expected/missing_text.html'),
109                 self.get_test('failures/expected/image.html'),
110                 self.get_test('passes/text.html')]
111
112     def get_basic_expectations(self):
113         return """
114 BUG_TEST : failures/expected/text.html = TEXT
115 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
116 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
117 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
118 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
119 """
120
121     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
122         test_config = self._port.test_configuration()
123         self._exp = TestExpectations(self._port,
124              tests=self.get_basic_tests(),
125              expectations=expectations,
126              test_config=test_config,
127              is_lint_mode=is_lint_mode,
128              overrides=overrides)
129
130     def assert_exp(self, test, result):
131         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
132                           set([result]))
133
134
135 class BasicTests(Base):
136     def test_basic(self):
137         self.parse_exp(self.get_basic_expectations())
138         self.assert_exp('failures/expected/text.html', TEXT)
139         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
140         self.assert_exp('passes/text.html', PASS)
141         self.assert_exp('failures/expected/image.html', PASS)
142
143
144 class MiscTests(Base):
145     def test_multiple_results(self):
146         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
147         self.assertEqual(self._exp.get_expectations(
148             self.get_test('failures/expected/text.html')),
149             set([TEXT, CRASH]))
150
151     def test_category_expectations(self):
152         # This test checks unknown tests are not present in the
153         # expectations and that known test part of a test category is
154         # present in the expectations.
155         exp_str = """
156 BUGX WONTFIX : failures/expected = IMAGE
157 """
158         self.parse_exp(exp_str)
159         test_name = 'failures/expected/unknown-test.html'
160         unknown_test = self.get_test(test_name)
161         self.assertRaises(KeyError, self._exp.get_expectations,
162                           unknown_test)
163         self.assert_exp('failures/expected/crash.html', IMAGE)
164
165     def test_get_modifiers(self):
166         self.parse_exp(self.get_basic_expectations())
167         self.assertEqual(self._exp.get_modifiers(
168                          self.get_test('passes/text.html')), [])
169
170     def test_get_expectations_string(self):
171         self.parse_exp(self.get_basic_expectations())
172         self.assertEquals(self._exp.get_expectations_string(
173                           self.get_test('failures/expected/text.html')),
174                           'TEXT')
175
176     def test_expectation_to_string(self):
177         # Normal cases are handled by other tests.
178         self.parse_exp(self.get_basic_expectations())
179         self.assertRaises(ValueError, self._exp.expectation_to_string,
180                           -1)
181
182     def test_get_test_set(self):
183         # Handle some corner cases for this routine not covered by other tests.
184         self.parse_exp(self.get_basic_expectations())
185         s = self._exp.get_test_set(WONTFIX)
186         self.assertEqual(s,
187             set([self.get_test('failures/expected/crash.html'),
188                  self.get_test('failures/expected/image_checksum.html')]))
189         s = self._exp.get_test_set(WONTFIX, CRASH)
190         self.assertEqual(s,
191             set([self.get_test('failures/expected/crash.html')]))
192         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
193         self.assertEqual(s, set([]))
194
195     def test_parse_error_fatal(self):
196         try:
197             self.parse_exp("""FOO : failures/expected/text.html = TEXT
198 SKIP : failures/expected/image.html""")
199             self.assertFalse(True, "ParseError wasn't raised")
200         except ParseError, e:
201             self.assertTrue(e.fatal)
202             exp_errors = [u"Line:1 Unrecognized modifier 'foo' failures/expected/text.html",
203                           u"Line:2 Missing expectations SKIP : failures/expected/image.html"]
204             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
205             self.assertEqual(e.errors, exp_errors)
206
207     def test_parse_error_nonfatal(self):
208         try:
209             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
210                            is_lint_mode=True)
211             self.assertFalse(True, "ParseError wasn't raised")
212         except ParseError, e:
213             self.assertFalse(e.fatal)
214             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
215             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
216             self.assertEqual(e.errors, exp_errors)
217
218     def test_overrides(self):
219         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
220                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
221         self.assert_exp('failures/expected/text.html', IMAGE)
222
223     def test_overrides__duplicate(self):
224         self.assertRaises(ParseError, self.parse_exp,
225              "BUG_EXP: failures/expected/text.html = TEXT",
226              """
227 BUG_OVERRIDE : failures/expected/text.html = IMAGE
228 BUG_OVERRIDE : failures/expected/text.html = CRASH
229 """)
230
231     def test_pixel_tests_flag(self):
232         def match(test, result, pixel_tests_enabled):
233             return self._exp.matches_an_expected_result(
234                 self.get_test(test), result, pixel_tests_enabled)
235
236         self.parse_exp(self.get_basic_expectations())
237         self.assertTrue(match('failures/expected/text.html', TEXT, True))
238         self.assertTrue(match('failures/expected/text.html', TEXT, False))
239         self.assertFalse(match('failures/expected/text.html', CRASH, True))
240         self.assertFalse(match('failures/expected/text.html', CRASH, False))
241         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
242                               True))
243         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
244                               False))
245         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
246         self.assertTrue(match('passes/text.html', PASS, False))
247
248     def test_more_specific_override_resets_skip(self):
249         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
250                        "BUGX : failures/expected/text.html = IMAGE\n")
251         self.assert_exp('failures/expected/text.html', IMAGE)
252         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
253                                                      'failures/expected/text.html') in
254                          self._exp.get_tests_with_result_type(SKIP))
255
256 class ExpectationSyntaxTests(Base):
257     def test_missing_expectation(self):
258         # This is missing the expectation.
259         self.assertRaises(ParseError, self.parse_exp,
260                           'BUG_TEST: failures/expected/text.html')
261
262     def test_missing_colon(self):
263         # This is missing the modifiers and the ':'
264         self.assertRaises(ParseError, self.parse_exp,
265                           'failures/expected/text.html = TEXT')
266
267     def disabled_test_too_many_colons(self):
268         # FIXME: Enable this test and fix the underlying bug.
269         self.assertRaises(ParseError, self.parse_exp,
270                           'BUG_TEST: failures/expected/text.html = PASS :')
271
272     def test_too_many_equals_signs(self):
273         self.assertRaises(ParseError, self.parse_exp,
274                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
275
276     def test_unrecognized_expectation(self):
277         self.assertRaises(ParseError, self.parse_exp,
278                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
279
280     def test_macro(self):
281         exp_str = """
282 BUG_TEST WIN : failures/expected/text.html = TEXT
283 """
284         self.parse_exp(exp_str)
285         self.assert_exp('failures/expected/text.html', TEXT)
286
287
288 class SemanticTests(Base):
289     def test_bug_format(self):
290         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
291
292     def test_missing_bugid(self):
293         # This should log a non-fatal error.
294         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
295         self.assertTrue(self._exp.has_warnings())
296
297     def test_slow_and_timeout(self):
298         # A test cannot be SLOW and expected to TIMEOUT.
299         self.assertRaises(ParseError, self.parse_exp,
300             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
301
302     def test_rebaseline(self):
303         # Can't lint a file w/ 'REBASELINE' in it.
304         self.assertRaises(ParseError, self.parse_exp,
305             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
306             is_lint_mode=True)
307
308     def test_duplicates(self):
309         self.assertRaises(ParseError, self.parse_exp, """
310 BUG_EXP : failures/expected/text.html = TEXT
311 BUG_EXP : failures/expected/text.html = IMAGE""")
312
313         self.assertRaises(ParseError, self.parse_exp,
314             self.get_basic_expectations(), overrides="""
315 BUG_OVERRIDE : failures/expected/text.html = TEXT
316 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
317
318     def test_missing_file(self):
319         # This should log a non-fatal error.
320         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
321         self.assertTrue(self._exp.has_warnings(), 1)
322
323
324 class PrecedenceTests(Base):
325     def test_file_over_directory(self):
326         # This tests handling precedence of specific lines over directories
327         # and tests expectations covering entire directories.
328         exp_str = """
329 BUGX : failures/expected/text.html = TEXT
330 BUGX WONTFIX : failures/expected = IMAGE
331 """
332         self.parse_exp(exp_str)
333         self.assert_exp('failures/expected/text.html', TEXT)
334         self.assert_exp('failures/expected/crash.html', IMAGE)
335
336         exp_str = """
337 BUGX WONTFIX : failures/expected = IMAGE
338 BUGX : failures/expected/text.html = TEXT
339 """
340         self.parse_exp(exp_str)
341         self.assert_exp('failures/expected/text.html', TEXT)
342         self.assert_exp('failures/expected/crash.html', IMAGE)
343
344     def test_ambiguous(self):
345         self.assertRaises(ParseError, self.parse_exp, """
346 BUG_TEST RELEASE : passes/text.html = PASS
347 BUG_TEST WIN : passes/text.html = FAIL
348 """)
349
350     def test_more_modifiers(self):
351         exp_str = """
352 BUG_TEST RELEASE : passes/text.html = PASS
353 BUG_TEST WIN RELEASE : passes/text.html = TEXT
354 """
355         self.assertRaises(ParseError, self.parse_exp, exp_str)
356
357     def test_order_in_file(self):
358         exp_str = """
359 BUG_TEST WIN RELEASE : passes/text.html = TEXT
360 BUG_TEST RELEASE : passes/text.html = PASS
361 """
362         self.assertRaises(ParseError, self.parse_exp, exp_str)
363
364     def test_macro_overrides(self):
365         exp_str = """
366 BUG_TEST WIN : passes/text.html = PASS
367 BUG_TEST XP : passes/text.html = TEXT
368 """
369         self.assertRaises(ParseError, self.parse_exp, exp_str)
370
371
372 class RebaseliningTest(Base):
373     """Test rebaselining-specific functionality."""
374     def assertRemove(self, input_expectations, tests, expected_expectations):
375         self.parse_exp(input_expectations)
376         actual_expectations = self._exp.remove_rebaselined_tests(tests)
377         self.assertEqual(expected_expectations, actual_expectations)
378
379     def test_remove(self):
380         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
381                           'BUGY : failures/expected/image.html = IMAGE\n'
382                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
383                           ['failures/expected/text.html'],
384                           'BUGY : failures/expected/image.html = IMAGE\n'
385                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
386
387     def test_no_get_rebaselining_failures(self):
388         self.parse_exp(self.get_basic_expectations())
389         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
390
391
392 class TestExpectationParserTests(unittest.TestCase):
393     def test_tokenize_blank(self):
394         expectation = TestExpectationParser.tokenize('')
395         self.assertEqual(expectation.is_malformed(), False)
396         self.assertEqual(expectation.comment, None)
397         self.assertEqual(len(expectation.errors), 0)
398
399     def test_tokenize_missing_colon(self):
400         expectation = TestExpectationParser.tokenize('Qux.')
401         self.assertEqual(expectation.is_malformed(), True)
402         self.assertEqual(str(expectation.errors), '["Missing a \':\'"]')
403
404     def test_tokenize_extra_colon(self):
405         expectation = TestExpectationParser.tokenize('FOO : : bar')
406         self.assertEqual(expectation.is_malformed(), True)
407         self.assertEqual(str(expectation.errors), '["Extraneous \':\'"]')
408
409     def test_tokenize_empty_comment(self):
410         expectation = TestExpectationParser.tokenize('//')
411         self.assertEqual(expectation.is_malformed(), False)
412         self.assertEqual(expectation.comment, '')
413         self.assertEqual(len(expectation.errors), 0)
414
415     def test_tokenize_comment(self):
416         expectation = TestExpectationParser.tokenize('//Qux.')
417         self.assertEqual(expectation.is_malformed(), False)
418         self.assertEqual(expectation.comment, 'Qux.')
419         self.assertEqual(len(expectation.errors), 0)
420
421     def test_tokenize_missing_equal(self):
422         expectation = TestExpectationParser.tokenize('FOO : bar')
423         self.assertEqual(expectation.is_malformed(), True)
424         self.assertEqual(str(expectation.errors), "['Missing expectations\']")
425
426     def test_tokenize_extra_equal(self):
427         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ = Qux.')
428         self.assertEqual(expectation.is_malformed(), True)
429         self.assertEqual(str(expectation.errors), '["Extraneous \'=\'"]')
430
431     def test_tokenize_valid(self):
432         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ')
433         self.assertEqual(expectation.is_malformed(), False)
434         self.assertEqual(expectation.comment, None)
435         self.assertEqual(len(expectation.errors), 0)
436
437     def test_tokenize_valid_with_comment(self):
438         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ //Qux.')
439         self.assertEqual(expectation.is_malformed(), False)
440         self.assertEqual(expectation.comment, 'Qux.')
441         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
442         self.assertEqual(str(expectation.expectations), '[\'baz\']')
443         self.assertEqual(len(expectation.errors), 0)
444
445     def test_tokenize_valid_with_multiple_modifiers(self):
446         expectation = TestExpectationParser.tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
447         self.assertEqual(expectation.is_malformed(), False)
448         self.assertEqual(expectation.comment, 'Qux.')
449         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
450         self.assertEqual(str(expectation.expectations), '[\'baz\']')
451         self.assertEqual(len(expectation.errors), 0)
452
453     def test_parse_empty_string(self):
454         test_port = port.get('test-win-xp', None)
455         test_port.test_exists = lambda test: True
456         test_config = test_port.test_configuration()
457         full_test_list = []
458         expectation_line = TestExpectationParser.tokenize('')
459         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
460         parser.parse(expectation_line)
461         self.assertFalse(expectation_line.is_invalid())
462
463
464 class TestExpectationSerializerTests(unittest.TestCase):
465     def __init__(self, testFunc):
466         test_port = port.get('test-win-xp', None)
467         self._serializer = TestExpectationSerializer(TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros()))
468         unittest.TestCase.__init__(self, testFunc)
469
470     def assert_round_trip(self, in_string, expected_string=None):
471         expectation = TestExpectationParser.tokenize(in_string)
472         if expected_string is None:
473             expected_string = in_string
474         self.assertEqual(expected_string, self._serializer.to_string(expectation))
475
476     def assert_list_round_trip(self, in_string, expected_string=None):
477         expectations = TestExpectationParser.tokenize_list(in_string)
478         if expected_string is None:
479             expected_string = in_string
480         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, SpecifierSorter()))
481
482     def test_unparsed_to_string(self):
483         expectation = TestExpectationLine()
484
485         self.assertEqual(self._serializer.to_string(expectation), '')
486         expectation.comment = 'Qux.'
487         self.assertEqual(self._serializer.to_string(expectation), '//Qux.')
488         expectation.name = 'bar'
489         self.assertEqual(self._serializer.to_string(expectation), ' : bar =  //Qux.')
490         expectation.modifiers = ['foo']
491         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar =  //Qux.')
492         expectation.expectations = ['bAz']
493         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
494         expectation.expectations = ['bAz1', 'baZ2']
495         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
496         expectation.modifiers = ['foo1', 'foO2']
497         self.assertEqual(self._serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
498         expectation.errors.append('Oh the horror.')
499         self.assertEqual(self._serializer.to_string(expectation), '')
500         expectation.original_string = 'Yes it is!'
501         self.assertEqual(self._serializer.to_string(expectation), 'Yes it is!')
502
503     def test_parsed_to_string(self):
504         expectation_line = TestExpectationLine()
505         expectation_line.parsed_bug_modifiers = ['BUGX']
506         expectation_line.name = 'test/name/for/realz.html'
507         expectation_line.parsed_expectations = set([IMAGE])
508         self.assertEqual(self._serializer.to_string(expectation_line), None)
509         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')])
510         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE')
511         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')])
512         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
513         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu')])
514         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE\nBUGX XP DEBUG GPU : test/name/for/realz.html = IMAGE')
515
516     def test_parsed_expectations_string(self):
517         expectation_line = TestExpectationLine()
518         expectation_line.parsed_expectations = set([])
519         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
520         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
521         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
522         expectation_line.parsed_expectations = set([PASS, FAIL])
523         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
524         expectation_line.parsed_expectations = set([FAIL, PASS])
525         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
526
527     def test_parsed_modifier_string(self):
528         expectation_line = TestExpectationLine()
529         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
530         expectation_line.parsed_modifiers = ['for', 'the']
531         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
532         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
533         expectation_line.parsed_bug_modifiers = []
534         expectation_line.parsed_modifiers = []
535         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
536         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
537         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
538         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
539         expectation_line.parsed_bug_modifiers = []
540         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
541         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
542
543
544     def test_format_result(self):
545         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
546         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
547
548     def test_string_roundtrip(self):
549         self.assert_round_trip('')
550         self.assert_round_trip('FOO')
551         self.assert_round_trip(':')
552         self.assert_round_trip('FOO :')
553         self.assert_round_trip('FOO : bar')
554         self.assert_round_trip('  FOO :')
555         self.assert_round_trip('    FOO : bar')
556         self.assert_round_trip('FOO : bar = BAZ')
557         self.assert_round_trip('FOO : bar = BAZ //Qux.')
558         self.assert_round_trip('FOO : bar = BAZ // Qux.')
559         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
560         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
561         self.assert_round_trip('FOO : : bar = BAZ')
562         self.assert_round_trip('FOO : : bar = BAZ')
563         self.assert_round_trip('FOO : : bar ==== BAZ')
564         self.assert_round_trip('=')
565         self.assert_round_trip('//')
566         self.assert_round_trip('// ')
567         self.assert_round_trip('// Foo')
568         self.assert_round_trip('// Foo')
569         self.assert_round_trip('// Foo :')
570         self.assert_round_trip('// Foo : =')
571
572     def test_list_roundtrip(self):
573         self.assert_list_round_trip('')
574         self.assert_list_round_trip('\n')
575         self.assert_list_round_trip('\n\n')
576         self.assert_list_round_trip('bar')
577         self.assert_list_round_trip('bar\n//Qux.')
578         self.assert_list_round_trip('bar\n//Qux.\n')
579
580     def test_string_whitespace_stripping(self):
581         self.assert_round_trip('\n', '')
582         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
583         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
584         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
585         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
586         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
587         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
588
589
590 class TestExpectationEditorTests(unittest.TestCase):
591     WIN_RELEASE_CPU_CONFIGS = set([
592         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
593         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
594         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
595     ])
596
597     RELEASE_CONFIGS = set([
598         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
599         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
600         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
601         TestConfiguration(None, 'vista', 'x86', 'release', 'gpu'),
602         TestConfiguration(None, 'win7', 'x86', 'release', 'gpu'),
603         TestConfiguration(None, 'xp', 'x86', 'release', 'gpu'),
604         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'cpu'),
605         TestConfiguration(None, 'leopard', 'x86', 'release', 'cpu'),
606         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'gpu'),
607         TestConfiguration(None, 'leopard', 'x86', 'release', 'gpu'),
608         TestConfiguration(None, 'lucid', 'x86', 'release', 'cpu'),
609         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'cpu'),
610         TestConfiguration(None, 'lucid', 'x86', 'release', 'gpu'),
611         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'gpu'),
612     ])
613
614     def __init__(self, testFunc):
615         self.test_port = port.get('test-win-xp', None)
616         self.full_test_list = ['failures/expected/keyboard.html', 'failures/expected/audio.html']
617         unittest.TestCase.__init__(self, testFunc)
618
619     def make_parsed_expectation_lines(self, in_string):
620         expectation_lines = TestExpectationParser.tokenize_list(in_string)
621         parser = TestExpectationParser(self.test_port, self.full_test_list, allow_rebaseline_modifier=False)
622         for expectation_line in expectation_lines:
623             self.assertFalse(expectation_line.is_invalid())
624             parser.parse(expectation_line)
625         return expectation_lines
626
627     def assert_remove_roundtrip(self, in_string, test, expected_string, remove_flakes=False):
628         test_config_set = set([self.test_port.test_configuration()])
629         expectation_lines = self.make_parsed_expectation_lines(in_string)
630         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
631         editor.remove_expectation(test, test_config_set, remove_flakes)
632         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
633         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
634         self.assertEquals(result, expected_string)
635
636     def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, expected_update_count, remove_flakes=False, parsed_bug_modifiers=None, test_configs=None):
637         test_config_set = test_configs or set([self.test_port.test_configuration()])
638         expectation_lines = self.make_parsed_expectation_lines(in_string)
639         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
640         updated_expectation_lines = editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifiers=parsed_bug_modifiers)
641         for updated_expectation_line in updated_expectation_lines:
642             self.assertTrue(updated_expectation_line in expectation_lines)
643         self.assertEquals(len(updated_expectation_lines), expected_update_count)
644         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
645         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
646         self.assertEquals(result, expected_string)
647
648     def test_remove_expectation(self):
649         self.assert_remove_roundtrip("""
650 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
651 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/hang.html', """
652 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
653 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
654
655         self.assert_remove_roundtrip("""
656 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
657 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
658 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
659 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
660
661         self.assert_remove_roundtrip("""
662 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
663 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
664 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
665 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
666
667         self.assert_remove_roundtrip("""
668 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
669 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html',  """
670 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
671 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
672
673         self.assert_remove_roundtrip("""
674 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
675 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
676 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
677
678         self.assert_remove_roundtrip("""
679 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
680 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
681 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
682 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
683 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
684 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
685
686         self.assert_remove_roundtrip("""
687 BUGX1 XP : failures/expected/keyboard.html = IMAGE
688 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
689 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
690 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
691 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
692
693         self.assert_remove_roundtrip("""
694 BUGX1 : failures/expected/keyboard.html = IMAGE
695 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
696 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
697 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
698 BUGX1 LINUX MAC VISTA WIN7 : failures/expected/keyboard.html = IMAGE
699 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
700
701         self.assert_remove_roundtrip("""
702 BUGX1 WIN : failures/expected = PASS
703 BUGX2 XP RELEASE : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
704 BUGX1 WIN : failures/expected = PASS
705 BUGX2 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE""")
706
707         self.assert_remove_roundtrip("""
708 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
709 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
710 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
711
712         self.assert_remove_roundtrip("""
713 BUGX1 WIN : failures/expected = FAIL""", 'failures/expected/keyboard.html', """
714 BUGX1 WIN : failures/expected = FAIL""")
715
716         self.assert_remove_roundtrip("""
717 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
718 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
719 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = PASS IMAGE
720 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
721
722         self.assert_remove_roundtrip("""
723 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
724 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
725 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", remove_flakes=True)
726
727     def test_remove_expectation_multiple(self):
728         in_string = """
729 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
730 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
731         expectation_lines = self.make_parsed_expectation_lines(in_string)
732         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
733         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
734         test = "failures/expected/keyboard.html"
735
736         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]))
737         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
738 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
739 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
740 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
741 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
742
743         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]))
744         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
745 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
746 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
747 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
748
749         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'win7', 'x86', 'release', 'gpu')]))
750         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
751 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
752 BUGX1 WIN7 DEBUG GPU : failures/expected/keyboard.html = IMAGE
753 BUGX1 WIN7 CPU : failures/expected/keyboard.html = IMAGE
754 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
755 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
756 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
757
758         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')]))
759         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
760 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
761 BUGX1 WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
762 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
763 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
764 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
765
766         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'cpu'), TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'vista', 'x86', 'release', 'gpu')]))
767         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
768 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
769 BUGX1 VISTA WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
770 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
771
772         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
773         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
774 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
775
776         test = "failures/expected/audio.html"
777
778         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
779         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), "")
780
781     def test_update_expectation(self):
782         self.assert_update_roundtrip("""
783 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
784 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 1)
785
786         self.assert_update_roundtrip("""
787 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '', 1)
788
789         self.assert_update_roundtrip("""
790 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
791 BUGX1 XP RELEASE CPU : failures/expected = TEXT
792 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 1)
793
794         self.assert_update_roundtrip("""
795 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
796 BUGX1 XP RELEASE CPU : failures/expected = TEXT
797 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = PASS""", 1)
798
799         self.assert_update_roundtrip("""
800 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([TEXT]), """
801 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 0)
802
803         self.assert_update_roundtrip("""
804 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
805 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 1, parsed_bug_modifiers=['BUGAWESOME'])
806
807         self.assert_update_roundtrip("""
808 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
809 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
810 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2)
811
812         self.assert_update_roundtrip("""
813 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
814 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT""", 1)
815
816         self.assert_update_roundtrip("""
817 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
818 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
819 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2, parsed_bug_modifiers=['BUGAWESOME'])
820
821         self.assert_update_roundtrip("""
822 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
823 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
824 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
825 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
826 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2)
827
828         self.assert_update_roundtrip("""
829 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
830 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
831 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
832 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT""", 1)
833
834         self.assert_update_roundtrip("""
835 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
836 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
837 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
838 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
839 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2)
840
841         self.assert_update_roundtrip("""
842 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
843 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2, test_configs=self.WIN_RELEASE_CPU_CONFIGS)
844
845         self.assert_update_roundtrip("""
846 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '', 1, test_configs=self.WIN_RELEASE_CPU_CONFIGS)
847
848         self.assert_update_roundtrip("""
849 BUGX1 RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
850 BUGX1 LINUX MAC RELEASE CPU : failures/expected/keyboard.html = TEXT
851 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 2, test_configs=self.WIN_RELEASE_CPU_CONFIGS)
852
853         self.assert_update_roundtrip("""
854 BUGX1 MAC : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
855 BUGX1 MAC : failures/expected/keyboard.html = TEXT
856 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", 1, test_configs=self.WIN_RELEASE_CPU_CONFIGS)
857
858     def test_update_expectation_multiple(self):
859         in_string = """
860 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
861 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
862         expectation_lines = self.make_parsed_expectation_lines(in_string)
863         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
864         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
865         test = "failures/expected/keyboard.html"
866
867         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), ['BUG_UPDATE1'])
868         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
869 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
870 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
871 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
872 BUGX2 WIN : failures/expected/audio.html = IMAGE
873 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT""")
874
875         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), ['BUG_UPDATE2'])
876         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
877 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
878 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
879 BUGX2 WIN : failures/expected/audio.html = IMAGE
880 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT
881 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT""")
882
883         editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), ['BUG_UPDATE3'])
884         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
885 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
886 BUGX1 WIN7 RELEASE GPU : failures/expected/keyboard.html = IMAGE
887 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
888 BUGX1 VISTA XP GPU : failures/expected/keyboard.html = IMAGE
889 BUGX2 WIN : failures/expected/audio.html = IMAGE
890 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
891 BUG_UPDATE3 WIN RELEASE CPU : failures/expected/keyboard.html = CRASH""")
892
893         editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), ['BUG_UPDATE4'])
894         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
895 BUGX1 XP DEBUG GPU : failures/expected/keyboard.html = IMAGE
896 BUGX1 VISTA WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
897 BUGX2 WIN : failures/expected/audio.html = IMAGE
898 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
899 BUG_UPDATE4 RELEASE : failures/expected/keyboard.html = FAIL""")
900
901         editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), ['BUG_UPDATE5'])
902         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
903 BUGX2 WIN : failures/expected/audio.html = IMAGE
904 BUG_UPDATE5 : failures/expected/keyboard.html = TIMEOUT""")
905
906
907 if __name__ == '__main__':
908     unittest.main()