Add a way to edit test expectations.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.layout_tests import port
33 from webkitpy.layout_tests.port import base
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38
39 class MockBugManager(object):
40     def close_bug(self, bug_id, reference_bug_id=None):
41         pass
42
43     def create_bug(self):
44         return "BUG_NEWLY_CREATED"
45
46
47 class FunctionsTest(unittest.TestCase):
48     def test_result_was_expected(self):
49         # test basics
50         self.assertEquals(result_was_expected(PASS, set([PASS]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([PASS]),
53                                               False, False), False)
54
55         # test handling of FAIL expectations
56         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
59                                               False, False), True)
60         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
61                                               False, False), True)
62         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
63                                               False, False), False)
64
65         # test handling of SKIPped tests and results
66         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
67                                               False, True), True)
68         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
69                                               False, False), False)
70
71         # test handling of MISSING results and the REBASELINE modifier
72         self.assertEquals(result_was_expected(MISSING, set([PASS]),
73                                               True, False), True)
74         self.assertEquals(result_was_expected(MISSING, set([PASS]),
75                                               False, False), False)
76
77     def test_remove_pixel_failures(self):
78         self.assertEquals(remove_pixel_failures(set([TEXT])),
79                           set([TEXT]))
80         self.assertEquals(remove_pixel_failures(set([PASS])),
81                           set([PASS]))
82         self.assertEquals(remove_pixel_failures(set([IMAGE])),
83                           set([PASS]))
84         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
87                           set([PASS, CRASH]))
88
89
90 class Base(unittest.TestCase):
91     # Note that all of these tests are written assuming the configuration
92     # being tested is Windows XP, Release build.
93
94     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
95         self._port = port.get('test-win-xp', None)
96         self._fs = self._port._filesystem
97         self._exp = None
98         unittest.TestCase.__init__(self, testFunc)
99
100     def get_test(self, test_name):
101         # FIXME: Remove this routine and just reference test names directly.
102         return test_name
103
104     def get_basic_tests(self):
105         return [self.get_test('failures/expected/text.html'),
106                 self.get_test('failures/expected/image_checksum.html'),
107                 self.get_test('failures/expected/crash.html'),
108                 self.get_test('failures/expected/missing_text.html'),
109                 self.get_test('failures/expected/image.html'),
110                 self.get_test('passes/text.html')]
111
112     def get_basic_expectations(self):
113         return """
114 BUG_TEST : failures/expected/text.html = TEXT
115 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
116 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
117 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
118 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
119 """
120
121     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
122         test_config = self._port.test_configuration()
123         self._exp = TestExpectations(self._port,
124              tests=self.get_basic_tests(),
125              expectations=expectations,
126              test_config=test_config,
127              is_lint_mode=is_lint_mode,
128              overrides=overrides)
129
130     def assert_exp(self, test, result):
131         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
132                           set([result]))
133
134
135 class BasicTests(Base):
136     def test_basic(self):
137         self.parse_exp(self.get_basic_expectations())
138         self.assert_exp('failures/expected/text.html', TEXT)
139         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
140         self.assert_exp('passes/text.html', PASS)
141         self.assert_exp('failures/expected/image.html', PASS)
142
143
144 class MiscTests(Base):
145     def test_multiple_results(self):
146         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
147         self.assertEqual(self._exp.get_expectations(
148             self.get_test('failures/expected/text.html')),
149             set([TEXT, CRASH]))
150
151     def test_category_expectations(self):
152         # This test checks unknown tests are not present in the
153         # expectations and that known test part of a test category is
154         # present in the expectations.
155         exp_str = """
156 BUGX WONTFIX : failures/expected = IMAGE
157 """
158         self.parse_exp(exp_str)
159         test_name = 'failures/expected/unknown-test.html'
160         unknown_test = self.get_test(test_name)
161         self.assertRaises(KeyError, self._exp.get_expectations,
162                           unknown_test)
163         self.assert_exp('failures/expected/crash.html', IMAGE)
164
165     def test_get_modifiers(self):
166         self.parse_exp(self.get_basic_expectations())
167         self.assertEqual(self._exp.get_modifiers(
168                          self.get_test('passes/text.html')), [])
169
170     def test_get_expectations_string(self):
171         self.parse_exp(self.get_basic_expectations())
172         self.assertEquals(self._exp.get_expectations_string(
173                           self.get_test('failures/expected/text.html')),
174                           'TEXT')
175
176     def test_expectation_to_string(self):
177         # Normal cases are handled by other tests.
178         self.parse_exp(self.get_basic_expectations())
179         self.assertRaises(ValueError, self._exp.expectation_to_string,
180                           -1)
181
182     def test_get_test_set(self):
183         # Handle some corner cases for this routine not covered by other tests.
184         self.parse_exp(self.get_basic_expectations())
185         s = self._exp.get_test_set(WONTFIX)
186         self.assertEqual(s,
187             set([self.get_test('failures/expected/crash.html'),
188                  self.get_test('failures/expected/image_checksum.html')]))
189         s = self._exp.get_test_set(WONTFIX, CRASH)
190         self.assertEqual(s,
191             set([self.get_test('failures/expected/crash.html')]))
192         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
193         self.assertEqual(s, set([]))
194
195     def test_parse_error_fatal(self):
196         try:
197             self.parse_exp("""FOO : failures/expected/text.html = TEXT
198 SKIP : failures/expected/image.html""")
199             self.assertFalse(True, "ParseError wasn't raised")
200         except ParseError, e:
201             self.assertTrue(e.fatal)
202             exp_errors = [u"Line:1 Unrecognized modifier 'foo' failures/expected/text.html",
203                           u"Line:2 Missing expectations SKIP : failures/expected/image.html"]
204             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
205             self.assertEqual(e.errors, exp_errors)
206
207     def test_parse_error_nonfatal(self):
208         try:
209             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
210                            is_lint_mode=True)
211             self.assertFalse(True, "ParseError wasn't raised")
212         except ParseError, e:
213             self.assertFalse(e.fatal)
214             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
215             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
216             self.assertEqual(e.errors, exp_errors)
217
218     def test_overrides(self):
219         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
220                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
221         self.assert_exp('failures/expected/text.html', IMAGE)
222
223     def test_overrides__duplicate(self):
224         self.assertRaises(ParseError, self.parse_exp,
225              "BUG_EXP: failures/expected/text.html = TEXT",
226              """
227 BUG_OVERRIDE : failures/expected/text.html = IMAGE
228 BUG_OVERRIDE : failures/expected/text.html = CRASH
229 """)
230
231     def test_pixel_tests_flag(self):
232         def match(test, result, pixel_tests_enabled):
233             return self._exp.matches_an_expected_result(
234                 self.get_test(test), result, pixel_tests_enabled)
235
236         self.parse_exp(self.get_basic_expectations())
237         self.assertTrue(match('failures/expected/text.html', TEXT, True))
238         self.assertTrue(match('failures/expected/text.html', TEXT, False))
239         self.assertFalse(match('failures/expected/text.html', CRASH, True))
240         self.assertFalse(match('failures/expected/text.html', CRASH, False))
241         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
242                               True))
243         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
244                               False))
245         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
246         self.assertTrue(match('passes/text.html', PASS, False))
247
248     def test_more_specific_override_resets_skip(self):
249         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
250                        "BUGX : failures/expected/text.html = IMAGE\n")
251         self.assert_exp('failures/expected/text.html', IMAGE)
252         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
253                                                      'failures/expected/text.html') in
254                          self._exp.get_tests_with_result_type(SKIP))
255
256 class ExpectationSyntaxTests(Base):
257     def test_missing_expectation(self):
258         # This is missing the expectation.
259         self.assertRaises(ParseError, self.parse_exp,
260                           'BUG_TEST: failures/expected/text.html')
261
262     def test_missing_colon(self):
263         # This is missing the modifiers and the ':'
264         self.assertRaises(ParseError, self.parse_exp,
265                           'failures/expected/text.html = TEXT')
266
267     def disabled_test_too_many_colons(self):
268         # FIXME: Enable this test and fix the underlying bug.
269         self.assertRaises(ParseError, self.parse_exp,
270                           'BUG_TEST: failures/expected/text.html = PASS :')
271
272     def test_too_many_equals_signs(self):
273         self.assertRaises(ParseError, self.parse_exp,
274                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
275
276     def test_unrecognized_expectation(self):
277         self.assertRaises(ParseError, self.parse_exp,
278                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
279
280     def test_macro(self):
281         exp_str = """
282 BUG_TEST WIN : failures/expected/text.html = TEXT
283 """
284         self.parse_exp(exp_str)
285         self.assert_exp('failures/expected/text.html', TEXT)
286
287
288 class SemanticTests(Base):
289     def test_bug_format(self):
290         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
291
292     def test_missing_bugid(self):
293         # This should log a non-fatal error.
294         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
295         self.assertTrue(self._exp.has_warnings())
296
297     def test_slow_and_timeout(self):
298         # A test cannot be SLOW and expected to TIMEOUT.
299         self.assertRaises(ParseError, self.parse_exp,
300             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
301
302     def test_rebaseline(self):
303         # Can't lint a file w/ 'REBASELINE' in it.
304         self.assertRaises(ParseError, self.parse_exp,
305             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
306             is_lint_mode=True)
307
308     def test_duplicates(self):
309         self.assertRaises(ParseError, self.parse_exp, """
310 BUG_EXP : failures/expected/text.html = TEXT
311 BUG_EXP : failures/expected/text.html = IMAGE""")
312
313         self.assertRaises(ParseError, self.parse_exp,
314             self.get_basic_expectations(), overrides="""
315 BUG_OVERRIDE : failures/expected/text.html = TEXT
316 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
317
318     def test_missing_file(self):
319         # This should log a non-fatal error.
320         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
321         self.assertTrue(self._exp.has_warnings(), 1)
322
323
324 class PrecedenceTests(Base):
325     def test_file_over_directory(self):
326         # This tests handling precedence of specific lines over directories
327         # and tests expectations covering entire directories.
328         exp_str = """
329 BUGX : failures/expected/text.html = TEXT
330 BUGX WONTFIX : failures/expected = IMAGE
331 """
332         self.parse_exp(exp_str)
333         self.assert_exp('failures/expected/text.html', TEXT)
334         self.assert_exp('failures/expected/crash.html', IMAGE)
335
336         exp_str = """
337 BUGX WONTFIX : failures/expected = IMAGE
338 BUGX : failures/expected/text.html = TEXT
339 """
340         self.parse_exp(exp_str)
341         self.assert_exp('failures/expected/text.html', TEXT)
342         self.assert_exp('failures/expected/crash.html', IMAGE)
343
344     def test_ambiguous(self):
345         self.assertRaises(ParseError, self.parse_exp, """
346 BUG_TEST RELEASE : passes/text.html = PASS
347 BUG_TEST WIN : passes/text.html = FAIL
348 """)
349
350     def test_more_modifiers(self):
351         exp_str = """
352 BUG_TEST RELEASE : passes/text.html = PASS
353 BUG_TEST WIN RELEASE : passes/text.html = TEXT
354 """
355         self.assertRaises(ParseError, self.parse_exp, exp_str)
356
357     def test_order_in_file(self):
358         exp_str = """
359 BUG_TEST WIN RELEASE : passes/text.html = TEXT
360 BUG_TEST RELEASE : passes/text.html = PASS
361 """
362         self.assertRaises(ParseError, self.parse_exp, exp_str)
363
364     def test_macro_overrides(self):
365         exp_str = """
366 BUG_TEST WIN : passes/text.html = PASS
367 BUG_TEST XP : passes/text.html = TEXT
368 """
369         self.assertRaises(ParseError, self.parse_exp, exp_str)
370
371
372 class RebaseliningTest(Base):
373     """Test rebaselining-specific functionality."""
374     def assertRemove(self, input_expectations, tests, expected_expectations):
375         self.parse_exp(input_expectations)
376         actual_expectations = self._exp.remove_rebaselined_tests(tests)
377         self.assertEqual(expected_expectations, actual_expectations)
378
379     def test_remove(self):
380         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
381                           'BUGY : failures/expected/image.html = IMAGE\n'
382                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
383                           ['failures/expected/text.html'],
384                           'BUGY : failures/expected/image.html = IMAGE\n'
385                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
386
387     def test_no_get_rebaselining_failures(self):
388         self.parse_exp(self.get_basic_expectations())
389         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
390
391
392 class TestExpectationParserTests(unittest.TestCase):
393     def test_tokenize_blank(self):
394         expectation = TestExpectationParser.tokenize('')
395         self.assertEqual(expectation.is_malformed(), False)
396         self.assertEqual(expectation.comment, None)
397         self.assertEqual(len(expectation.errors), 0)
398
399     def test_tokenize_missing_colon(self):
400         expectation = TestExpectationParser.tokenize('Qux.')
401         self.assertEqual(expectation.is_malformed(), True)
402         self.assertEqual(str(expectation.errors), '["Missing a \':\'"]')
403
404     def test_tokenize_extra_colon(self):
405         expectation = TestExpectationParser.tokenize('FOO : : bar')
406         self.assertEqual(expectation.is_malformed(), True)
407         self.assertEqual(str(expectation.errors), '["Extraneous \':\'"]')
408
409     def test_tokenize_empty_comment(self):
410         expectation = TestExpectationParser.tokenize('//')
411         self.assertEqual(expectation.is_malformed(), False)
412         self.assertEqual(expectation.comment, '')
413         self.assertEqual(len(expectation.errors), 0)
414
415     def test_tokenize_comment(self):
416         expectation = TestExpectationParser.tokenize('//Qux.')
417         self.assertEqual(expectation.is_malformed(), False)
418         self.assertEqual(expectation.comment, 'Qux.')
419         self.assertEqual(len(expectation.errors), 0)
420
421     def test_tokenize_missing_equal(self):
422         expectation = TestExpectationParser.tokenize('FOO : bar')
423         self.assertEqual(expectation.is_malformed(), True)
424         self.assertEqual(str(expectation.errors), "['Missing expectations\']")
425
426     def test_tokenize_extra_equal(self):
427         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ = Qux.')
428         self.assertEqual(expectation.is_malformed(), True)
429         self.assertEqual(str(expectation.errors), '["Extraneous \'=\'"]')
430
431     def test_tokenize_valid(self):
432         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ')
433         self.assertEqual(expectation.is_malformed(), False)
434         self.assertEqual(expectation.comment, None)
435         self.assertEqual(len(expectation.errors), 0)
436
437     def test_tokenize_valid_with_comment(self):
438         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ //Qux.')
439         self.assertEqual(expectation.is_malformed(), False)
440         self.assertEqual(expectation.comment, 'Qux.')
441         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
442         self.assertEqual(str(expectation.expectations), '[\'baz\']')
443         self.assertEqual(len(expectation.errors), 0)
444
445     def test_tokenize_valid_with_multiple_modifiers(self):
446         expectation = TestExpectationParser.tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
447         self.assertEqual(expectation.is_malformed(), False)
448         self.assertEqual(expectation.comment, 'Qux.')
449         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
450         self.assertEqual(str(expectation.expectations), '[\'baz\']')
451         self.assertEqual(len(expectation.errors), 0)
452
453     def test_parse_empty_string(self):
454         test_port = port.get('test-win-xp', None)
455         test_port.test_exists = lambda test: True
456         test_config = test_port.test_configuration()
457         full_test_list = []
458         expectation_line = TestExpectationParser.tokenize('')
459         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
460         parser.parse(expectation_line)
461         self.assertFalse(expectation_line.is_invalid())
462
463
464 class TestExpectationSerializerTests(unittest.TestCase):
465     def __init__(self, testFunc):
466         test_port = port.get('test-win-xp', None)
467         self._serializer = TestExpectationSerializer(TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros()))
468         unittest.TestCase.__init__(self, testFunc)
469
470     def assert_round_trip(self, in_string, expected_string=None):
471         expectation = TestExpectationParser.tokenize(in_string)
472         if expected_string is None:
473             expected_string = in_string
474         self.assertEqual(expected_string, self._serializer.to_string(expectation))
475
476     def assert_list_round_trip(self, in_string, expected_string=None):
477         expectations = TestExpectationParser.tokenize_list(in_string)
478         if expected_string is None:
479             expected_string = in_string
480         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, SpecifierSorter()))
481
482     def test_unparsed_to_string(self):
483         expectation = TestExpectationLine()
484
485         self.assertEqual(self._serializer.to_string(expectation), '')
486         expectation.comment = 'Qux.'
487         self.assertEqual(self._serializer.to_string(expectation), '//Qux.')
488         expectation.name = 'bar'
489         self.assertEqual(self._serializer.to_string(expectation), ' : bar =  //Qux.')
490         expectation.modifiers = ['foo']
491         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar =  //Qux.')
492         expectation.expectations = ['bAz']
493         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
494         expectation.expectations = ['bAz1', 'baZ2']
495         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
496         expectation.modifiers = ['foo1', 'foO2']
497         self.assertEqual(self._serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
498         expectation.errors.append('Oh the horror.')
499         self.assertEqual(self._serializer.to_string(expectation), '')
500         expectation.original_string = 'Yes it is!'
501         self.assertEqual(self._serializer.to_string(expectation), 'Yes it is!')
502
503     def test_parsed_to_string(self):
504         expectation_line = TestExpectationLine()
505         expectation_line.parsed_bug_modifier = 'BUGX'
506         expectation_line.name = 'test/name/for/realz.html'
507         expectation_line.parsed_expectations = set([IMAGE])
508         self.assertEqual(self._serializer.to_string(expectation_line), None)
509         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')])
510         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE')
511         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')])
512         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
513         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu')])
514         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE\nBUGX XP DEBUG GPU : test/name/for/realz.html = IMAGE')
515
516     def test_parsed_expectations_string(self):
517         expectation_line = TestExpectationLine()
518         expectation_line.parsed_expectations = set([])
519         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
520         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
521         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
522         expectation_line.parsed_expectations = set([PASS, FAIL])
523         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
524         expectation_line.parsed_expectations = set([FAIL, PASS])
525         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
526
527     def test_parsed_modifier_string(self):
528         expectation_line = TestExpectationLine()
529         expectation_line.parsed_bug_modifier = 'garden-o-matic'
530         expectation_line.parsed_modifiers = ['for', 'the']
531         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
532         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
533         expectation_line.parsed_bug_modifier = ''
534         expectation_line.parsed_modifiers = []
535         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
536         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
537
538     def test_format_result(self):
539         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
540         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
541
542     def test_string_roundtrip(self):
543         self.assert_round_trip('')
544         self.assert_round_trip('FOO')
545         self.assert_round_trip(':')
546         self.assert_round_trip('FOO :')
547         self.assert_round_trip('FOO : bar')
548         self.assert_round_trip('  FOO :')
549         self.assert_round_trip('    FOO : bar')
550         self.assert_round_trip('FOO : bar = BAZ')
551         self.assert_round_trip('FOO : bar = BAZ //Qux.')
552         self.assert_round_trip('FOO : bar = BAZ // Qux.')
553         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
554         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
555         self.assert_round_trip('FOO : : bar = BAZ')
556         self.assert_round_trip('FOO : : bar = BAZ')
557         self.assert_round_trip('FOO : : bar ==== BAZ')
558         self.assert_round_trip('=')
559         self.assert_round_trip('//')
560         self.assert_round_trip('// ')
561         self.assert_round_trip('// Foo')
562         self.assert_round_trip('// Foo')
563         self.assert_round_trip('// Foo :')
564         self.assert_round_trip('// Foo : =')
565
566     def test_list_roundtrip(self):
567         self.assert_list_round_trip('')
568         self.assert_list_round_trip('\n')
569         self.assert_list_round_trip('\n\n')
570         self.assert_list_round_trip('bar')
571         self.assert_list_round_trip('bar\n//Qux.')
572         self.assert_list_round_trip('bar\n//Qux.\n')
573
574     def test_string_whitespace_stripping(self):
575         self.assert_round_trip('\n', '')
576         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
577         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
578         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
579         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
580         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
581         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
582
583
584 class TestExpectationEditorTests(unittest.TestCase):
585     WIN_RELEASE_CPU_CONFIGS = set([
586         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
587         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
588         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
589     ])
590
591     RELEASE_CONFIGS = set([
592         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
593         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
594         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
595         TestConfiguration(None, 'vista', 'x86', 'release', 'gpu'),
596         TestConfiguration(None, 'win7', 'x86', 'release', 'gpu'),
597         TestConfiguration(None, 'xp', 'x86', 'release', 'gpu'),
598         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'cpu'),
599         TestConfiguration(None, 'leopard', 'x86', 'release', 'cpu'),
600         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'gpu'),
601         TestConfiguration(None, 'leopard', 'x86', 'release', 'gpu'),
602         TestConfiguration(None, 'lucid', 'x86', 'release', 'cpu'),
603         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'cpu'),
604         TestConfiguration(None, 'lucid', 'x86', 'release', 'gpu'),
605         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'gpu'),
606     ])
607
608     def __init__(self, testFunc):
609         self.test_port = port.get('test-win-xp', None)
610         self.full_test_list = ['failures/expected/keyboard.html', 'failures/expected/audio.html']
611         unittest.TestCase.__init__(self, testFunc)
612
613     def make_parsed_expectation_lines(self, in_string):
614         expectation_lines = TestExpectationParser.tokenize_list(in_string)
615         parser = TestExpectationParser(self.test_port, self.full_test_list, allow_rebaseline_modifier=False)
616         for expectation_line in expectation_lines:
617             self.assertFalse(expectation_line.is_invalid())
618             parser.parse(expectation_line)
619         return expectation_lines
620
621     def assert_remove_roundtrip(self, in_string, test, expected_string, remove_flakes=False):
622         test_config_set = set([self.test_port.test_configuration()])
623         expectation_lines = self.make_parsed_expectation_lines(in_string)
624         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
625         editor.remove_expectation(test, test_config_set, remove_flakes)
626         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
627         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
628         self.assertEquals(result, expected_string)
629
630     def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, remove_flakes=False, parsed_bug_modifier=None, test_configs=None):
631         test_config_set = test_configs or set([self.test_port.test_configuration()])
632         expectation_lines = self.make_parsed_expectation_lines(in_string)
633         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
634         editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifier=parsed_bug_modifier)
635         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
636         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
637         self.assertEquals(result, expected_string)
638
639     def test_remove_expectation(self):
640         self.assert_remove_roundtrip("""
641 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
642 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/hang.html', """
643 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
644 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
645
646         self.assert_remove_roundtrip("""
647 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
648 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
649 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
650 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
651
652         self.assert_remove_roundtrip("""
653 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
654 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
655 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
656 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
657
658         self.assert_remove_roundtrip("""
659 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
660 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html',  """
661 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
662 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
663
664         self.assert_remove_roundtrip("""
665 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
666 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
667 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
668
669         self.assert_remove_roundtrip("""
670 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
671 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
672 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
673 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
674 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
675 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
676
677         self.assert_remove_roundtrip("""
678 BUGX1 XP : failures/expected/keyboard.html = IMAGE
679 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
680 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
681 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
682 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
683
684         self.assert_remove_roundtrip("""
685 BUGX1 : failures/expected/keyboard.html = IMAGE
686 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
687 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
688 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
689 BUGX1 LINUX MAC VISTA WIN7 : failures/expected/keyboard.html = IMAGE
690 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
691
692         self.assert_remove_roundtrip("""
693 BUGX1 WIN : failures/expected = PASS
694 BUGX2 XP RELEASE : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
695 BUGX1 WIN : failures/expected = PASS
696 BUGX2 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE""")
697
698         self.assert_remove_roundtrip("""
699 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
700 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
701 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
702
703         self.assert_remove_roundtrip("""
704 BUGX1 WIN : failures/expected = FAIL""", 'failures/expected/keyboard.html', """
705 BUGX1 WIN : failures/expected = FAIL""")
706
707         self.assert_remove_roundtrip("""
708 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
709 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
710 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = PASS IMAGE
711 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
712
713         self.assert_remove_roundtrip("""
714 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
715 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
716 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", remove_flakes=True)
717
718     def test_remove_expectation_multiple(self):
719         in_string = """
720 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
721 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
722         expectation_lines = self.make_parsed_expectation_lines(in_string)
723         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
724         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
725         test = "failures/expected/keyboard.html"
726
727         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]))
728         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
729 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
730 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
731 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
732 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
733
734         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]))
735         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
736 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
737 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
738 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
739
740         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'win7', 'x86', 'release', 'gpu')]))
741         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
742 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
743 BUGX1 WIN7 DEBUG GPU : failures/expected/keyboard.html = IMAGE
744 BUGX1 WIN7 CPU : failures/expected/keyboard.html = IMAGE
745 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
746 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
747 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
748
749         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')]))
750         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
751 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
752 BUGX1 WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
753 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
754 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
755 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
756
757         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'cpu'), TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'vista', 'x86', 'release', 'gpu')]))
758         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
759 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
760 BUGX1 VISTA WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
761 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
762
763         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
764         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
765 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
766
767         test = "failures/expected/audio.html"
768
769         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
770         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), "")
771
772     def test_update_expectation(self):
773         self.assert_update_roundtrip("""
774 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
775 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
776
777         self.assert_update_roundtrip("""
778 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '')
779
780         self.assert_update_roundtrip("""
781 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
782 BUGX1 XP RELEASE CPU : failures/expected = TEXT
783 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
784
785         self.assert_update_roundtrip("""
786 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
787 BUGX1 XP RELEASE CPU : failures/expected = TEXT
788 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = PASS""")
789
790         self.assert_update_roundtrip("""
791 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([TEXT]), """
792 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""")
793
794         self.assert_update_roundtrip("""
795 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
796 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
797
798         self.assert_update_roundtrip("""
799 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
800 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
801 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
802
803         self.assert_update_roundtrip("""
804 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
805 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT""")
806
807         self.assert_update_roundtrip("""
808 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
809 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
810 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
811
812         self.assert_update_roundtrip("""
813 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
814 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
815 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
816 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
817 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
818
819         self.assert_update_roundtrip("""
820 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
821 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
822 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
823 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT""")
824
825         self.assert_update_roundtrip("""
826 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
827 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
828 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
829 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
830 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
831
832         self.assert_update_roundtrip("""
833 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
834 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
835
836         self.assert_update_roundtrip("""
837 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '', test_configs=self.WIN_RELEASE_CPU_CONFIGS)
838
839         self.assert_update_roundtrip("""
840 BUGX1 RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
841 BUGX1 LINUX MAC RELEASE CPU : failures/expected/keyboard.html = TEXT
842 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
843
844         self.assert_update_roundtrip("""
845 BUGX1 MAC : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
846 BUGX1 MAC : failures/expected/keyboard.html = TEXT
847 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
848
849     def test_update_expectation_multiple(self):
850         in_string = """
851 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
852 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
853         expectation_lines = self.make_parsed_expectation_lines(in_string)
854         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
855         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
856         test = "failures/expected/keyboard.html"
857
858         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), 'BUG_UPDATE1')
859         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
860 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
861 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
862 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
863 BUGX2 WIN : failures/expected/audio.html = IMAGE
864 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT""")
865
866         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), 'BUG_UPDATE2')
867         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
868 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
869 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
870 BUGX2 WIN : failures/expected/audio.html = IMAGE
871 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT
872 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT""")
873
874         editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), 'BUG_UPDATE3')
875         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
876 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
877 BUGX1 WIN7 RELEASE GPU : failures/expected/keyboard.html = IMAGE
878 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
879 BUGX1 VISTA XP GPU : failures/expected/keyboard.html = IMAGE
880 BUGX2 WIN : failures/expected/audio.html = IMAGE
881 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
882 BUG_UPDATE3 WIN RELEASE CPU : failures/expected/keyboard.html = CRASH""")
883
884         editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), 'BUG_UPDATE4')
885         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
886 BUGX1 XP DEBUG GPU : failures/expected/keyboard.html = IMAGE
887 BUGX1 VISTA WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
888 BUGX2 WIN : failures/expected/audio.html = IMAGE
889 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
890 BUG_UPDATE4 RELEASE : failures/expected/keyboard.html = FAIL""")
891
892         editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), 'BUG_UPDATE5')
893         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
894 BUGX2 WIN : failures/expected/audio.html = IMAGE
895 BUG_UPDATE5 : failures/expected/keyboard.html = TIMEOUT""")
896
897
898 if __name__ == '__main__':
899     unittest.main()