Allow multiple bug identifiers in test expectations.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.layout_tests import port
33 from webkitpy.layout_tests.port import base
34 from webkitpy.layout_tests.models.test_configuration import *
35 from webkitpy.layout_tests.models.test_expectations import *
36 from webkitpy.layout_tests.models.test_configuration import *
37
38
39 class MockBugManager(object):
40     def close_bug(self, bug_id, reference_bug_id=None):
41         pass
42
43     def create_bug(self):
44         return "BUG_NEWLY_CREATED"
45
46
47 class FunctionsTest(unittest.TestCase):
48     def test_result_was_expected(self):
49         # test basics
50         self.assertEquals(result_was_expected(PASS, set([PASS]),
51                                               False, False), True)
52         self.assertEquals(result_was_expected(TEXT, set([PASS]),
53                                               False, False), False)
54
55         # test handling of FAIL expectations
56         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
57                                               False, False), True)
58         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
59                                               False, False), True)
60         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
61                                               False, False), True)
62         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
63                                               False, False), False)
64
65         # test handling of SKIPped tests and results
66         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
67                                               False, True), True)
68         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
69                                               False, False), False)
70
71         # test handling of MISSING results and the REBASELINE modifier
72         self.assertEquals(result_was_expected(MISSING, set([PASS]),
73                                               True, False), True)
74         self.assertEquals(result_was_expected(MISSING, set([PASS]),
75                                               False, False), False)
76
77     def test_remove_pixel_failures(self):
78         self.assertEquals(remove_pixel_failures(set([TEXT])),
79                           set([TEXT]))
80         self.assertEquals(remove_pixel_failures(set([PASS])),
81                           set([PASS]))
82         self.assertEquals(remove_pixel_failures(set([IMAGE])),
83                           set([PASS]))
84         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
85                           set([TEXT]))
86         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
87                           set([PASS, CRASH]))
88
89
90 class Base(unittest.TestCase):
91     # Note that all of these tests are written assuming the configuration
92     # being tested is Windows XP, Release build.
93
94     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
95         self._port = port.get('test-win-xp', None)
96         self._fs = self._port._filesystem
97         self._exp = None
98         unittest.TestCase.__init__(self, testFunc)
99
100     def get_test(self, test_name):
101         # FIXME: Remove this routine and just reference test names directly.
102         return test_name
103
104     def get_basic_tests(self):
105         return [self.get_test('failures/expected/text.html'),
106                 self.get_test('failures/expected/image_checksum.html'),
107                 self.get_test('failures/expected/crash.html'),
108                 self.get_test('failures/expected/missing_text.html'),
109                 self.get_test('failures/expected/image.html'),
110                 self.get_test('passes/text.html')]
111
112     def get_basic_expectations(self):
113         return """
114 BUG_TEST : failures/expected/text.html = TEXT
115 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
116 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
117 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
118 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
119 """
120
121     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
122         test_config = self._port.test_configuration()
123         self._exp = TestExpectations(self._port,
124              tests=self.get_basic_tests(),
125              expectations=expectations,
126              test_config=test_config,
127              is_lint_mode=is_lint_mode,
128              overrides=overrides)
129
130     def assert_exp(self, test, result):
131         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
132                           set([result]))
133
134
135 class BasicTests(Base):
136     def test_basic(self):
137         self.parse_exp(self.get_basic_expectations())
138         self.assert_exp('failures/expected/text.html', TEXT)
139         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
140         self.assert_exp('passes/text.html', PASS)
141         self.assert_exp('failures/expected/image.html', PASS)
142
143
144 class MiscTests(Base):
145     def test_multiple_results(self):
146         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
147         self.assertEqual(self._exp.get_expectations(
148             self.get_test('failures/expected/text.html')),
149             set([TEXT, CRASH]))
150
151     def test_category_expectations(self):
152         # This test checks unknown tests are not present in the
153         # expectations and that known test part of a test category is
154         # present in the expectations.
155         exp_str = """
156 BUGX WONTFIX : failures/expected = IMAGE
157 """
158         self.parse_exp(exp_str)
159         test_name = 'failures/expected/unknown-test.html'
160         unknown_test = self.get_test(test_name)
161         self.assertRaises(KeyError, self._exp.get_expectations,
162                           unknown_test)
163         self.assert_exp('failures/expected/crash.html', IMAGE)
164
165     def test_get_modifiers(self):
166         self.parse_exp(self.get_basic_expectations())
167         self.assertEqual(self._exp.get_modifiers(
168                          self.get_test('passes/text.html')), [])
169
170     def test_get_expectations_string(self):
171         self.parse_exp(self.get_basic_expectations())
172         self.assertEquals(self._exp.get_expectations_string(
173                           self.get_test('failures/expected/text.html')),
174                           'TEXT')
175
176     def test_expectation_to_string(self):
177         # Normal cases are handled by other tests.
178         self.parse_exp(self.get_basic_expectations())
179         self.assertRaises(ValueError, self._exp.expectation_to_string,
180                           -1)
181
182     def test_get_test_set(self):
183         # Handle some corner cases for this routine not covered by other tests.
184         self.parse_exp(self.get_basic_expectations())
185         s = self._exp.get_test_set(WONTFIX)
186         self.assertEqual(s,
187             set([self.get_test('failures/expected/crash.html'),
188                  self.get_test('failures/expected/image_checksum.html')]))
189         s = self._exp.get_test_set(WONTFIX, CRASH)
190         self.assertEqual(s,
191             set([self.get_test('failures/expected/crash.html')]))
192         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
193         self.assertEqual(s, set([]))
194
195     def test_parse_error_fatal(self):
196         try:
197             self.parse_exp("""FOO : failures/expected/text.html = TEXT
198 SKIP : failures/expected/image.html""")
199             self.assertFalse(True, "ParseError wasn't raised")
200         except ParseError, e:
201             self.assertTrue(e.fatal)
202             exp_errors = [u"Line:1 Unrecognized modifier 'foo' failures/expected/text.html",
203                           u"Line:2 Missing expectations SKIP : failures/expected/image.html"]
204             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
205             self.assertEqual(e.errors, exp_errors)
206
207     def test_parse_error_nonfatal(self):
208         try:
209             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
210                            is_lint_mode=True)
211             self.assertFalse(True, "ParseError wasn't raised")
212         except ParseError, e:
213             self.assertFalse(e.fatal)
214             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
215             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
216             self.assertEqual(e.errors, exp_errors)
217
218     def test_overrides(self):
219         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
220                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
221         self.assert_exp('failures/expected/text.html', IMAGE)
222
223     def test_overrides__duplicate(self):
224         self.assertRaises(ParseError, self.parse_exp,
225              "BUG_EXP: failures/expected/text.html = TEXT",
226              """
227 BUG_OVERRIDE : failures/expected/text.html = IMAGE
228 BUG_OVERRIDE : failures/expected/text.html = CRASH
229 """)
230
231     def test_pixel_tests_flag(self):
232         def match(test, result, pixel_tests_enabled):
233             return self._exp.matches_an_expected_result(
234                 self.get_test(test), result, pixel_tests_enabled)
235
236         self.parse_exp(self.get_basic_expectations())
237         self.assertTrue(match('failures/expected/text.html', TEXT, True))
238         self.assertTrue(match('failures/expected/text.html', TEXT, False))
239         self.assertFalse(match('failures/expected/text.html', CRASH, True))
240         self.assertFalse(match('failures/expected/text.html', CRASH, False))
241         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
242                               True))
243         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
244                               False))
245         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
246         self.assertTrue(match('passes/text.html', PASS, False))
247
248     def test_more_specific_override_resets_skip(self):
249         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
250                        "BUGX : failures/expected/text.html = IMAGE\n")
251         self.assert_exp('failures/expected/text.html', IMAGE)
252         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
253                                                      'failures/expected/text.html') in
254                          self._exp.get_tests_with_result_type(SKIP))
255
256 class ExpectationSyntaxTests(Base):
257     def test_missing_expectation(self):
258         # This is missing the expectation.
259         self.assertRaises(ParseError, self.parse_exp,
260                           'BUG_TEST: failures/expected/text.html')
261
262     def test_missing_colon(self):
263         # This is missing the modifiers and the ':'
264         self.assertRaises(ParseError, self.parse_exp,
265                           'failures/expected/text.html = TEXT')
266
267     def disabled_test_too_many_colons(self):
268         # FIXME: Enable this test and fix the underlying bug.
269         self.assertRaises(ParseError, self.parse_exp,
270                           'BUG_TEST: failures/expected/text.html = PASS :')
271
272     def test_too_many_equals_signs(self):
273         self.assertRaises(ParseError, self.parse_exp,
274                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
275
276     def test_unrecognized_expectation(self):
277         self.assertRaises(ParseError, self.parse_exp,
278                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
279
280     def test_macro(self):
281         exp_str = """
282 BUG_TEST WIN : failures/expected/text.html = TEXT
283 """
284         self.parse_exp(exp_str)
285         self.assert_exp('failures/expected/text.html', TEXT)
286
287
288 class SemanticTests(Base):
289     def test_bug_format(self):
290         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
291
292     def test_missing_bugid(self):
293         # This should log a non-fatal error.
294         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
295         self.assertTrue(self._exp.has_warnings())
296
297     def test_slow_and_timeout(self):
298         # A test cannot be SLOW and expected to TIMEOUT.
299         self.assertRaises(ParseError, self.parse_exp,
300             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
301
302     def test_rebaseline(self):
303         # Can't lint a file w/ 'REBASELINE' in it.
304         self.assertRaises(ParseError, self.parse_exp,
305             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
306             is_lint_mode=True)
307
308     def test_duplicates(self):
309         self.assertRaises(ParseError, self.parse_exp, """
310 BUG_EXP : failures/expected/text.html = TEXT
311 BUG_EXP : failures/expected/text.html = IMAGE""")
312
313         self.assertRaises(ParseError, self.parse_exp,
314             self.get_basic_expectations(), overrides="""
315 BUG_OVERRIDE : failures/expected/text.html = TEXT
316 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
317
318     def test_missing_file(self):
319         # This should log a non-fatal error.
320         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
321         self.assertTrue(self._exp.has_warnings(), 1)
322
323
324 class PrecedenceTests(Base):
325     def test_file_over_directory(self):
326         # This tests handling precedence of specific lines over directories
327         # and tests expectations covering entire directories.
328         exp_str = """
329 BUGX : failures/expected/text.html = TEXT
330 BUGX WONTFIX : failures/expected = IMAGE
331 """
332         self.parse_exp(exp_str)
333         self.assert_exp('failures/expected/text.html', TEXT)
334         self.assert_exp('failures/expected/crash.html', IMAGE)
335
336         exp_str = """
337 BUGX WONTFIX : failures/expected = IMAGE
338 BUGX : failures/expected/text.html = TEXT
339 """
340         self.parse_exp(exp_str)
341         self.assert_exp('failures/expected/text.html', TEXT)
342         self.assert_exp('failures/expected/crash.html', IMAGE)
343
344     def test_ambiguous(self):
345         self.assertRaises(ParseError, self.parse_exp, """
346 BUG_TEST RELEASE : passes/text.html = PASS
347 BUG_TEST WIN : passes/text.html = FAIL
348 """)
349
350     def test_more_modifiers(self):
351         exp_str = """
352 BUG_TEST RELEASE : passes/text.html = PASS
353 BUG_TEST WIN RELEASE : passes/text.html = TEXT
354 """
355         self.assertRaises(ParseError, self.parse_exp, exp_str)
356
357     def test_order_in_file(self):
358         exp_str = """
359 BUG_TEST WIN RELEASE : passes/text.html = TEXT
360 BUG_TEST RELEASE : passes/text.html = PASS
361 """
362         self.assertRaises(ParseError, self.parse_exp, exp_str)
363
364     def test_macro_overrides(self):
365         exp_str = """
366 BUG_TEST WIN : passes/text.html = PASS
367 BUG_TEST XP : passes/text.html = TEXT
368 """
369         self.assertRaises(ParseError, self.parse_exp, exp_str)
370
371
372 class RebaseliningTest(Base):
373     """Test rebaselining-specific functionality."""
374     def assertRemove(self, input_expectations, tests, expected_expectations):
375         self.parse_exp(input_expectations)
376         actual_expectations = self._exp.remove_rebaselined_tests(tests)
377         self.assertEqual(expected_expectations, actual_expectations)
378
379     def test_remove(self):
380         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
381                           'BUGY : failures/expected/image.html = IMAGE\n'
382                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
383                           ['failures/expected/text.html'],
384                           'BUGY : failures/expected/image.html = IMAGE\n'
385                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
386
387     def test_no_get_rebaselining_failures(self):
388         self.parse_exp(self.get_basic_expectations())
389         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
390
391
392 class TestExpectationParserTests(unittest.TestCase):
393     def test_tokenize_blank(self):
394         expectation = TestExpectationParser.tokenize('')
395         self.assertEqual(expectation.is_malformed(), False)
396         self.assertEqual(expectation.comment, None)
397         self.assertEqual(len(expectation.errors), 0)
398
399     def test_tokenize_missing_colon(self):
400         expectation = TestExpectationParser.tokenize('Qux.')
401         self.assertEqual(expectation.is_malformed(), True)
402         self.assertEqual(str(expectation.errors), '["Missing a \':\'"]')
403
404     def test_tokenize_extra_colon(self):
405         expectation = TestExpectationParser.tokenize('FOO : : bar')
406         self.assertEqual(expectation.is_malformed(), True)
407         self.assertEqual(str(expectation.errors), '["Extraneous \':\'"]')
408
409     def test_tokenize_empty_comment(self):
410         expectation = TestExpectationParser.tokenize('//')
411         self.assertEqual(expectation.is_malformed(), False)
412         self.assertEqual(expectation.comment, '')
413         self.assertEqual(len(expectation.errors), 0)
414
415     def test_tokenize_comment(self):
416         expectation = TestExpectationParser.tokenize('//Qux.')
417         self.assertEqual(expectation.is_malformed(), False)
418         self.assertEqual(expectation.comment, 'Qux.')
419         self.assertEqual(len(expectation.errors), 0)
420
421     def test_tokenize_missing_equal(self):
422         expectation = TestExpectationParser.tokenize('FOO : bar')
423         self.assertEqual(expectation.is_malformed(), True)
424         self.assertEqual(str(expectation.errors), "['Missing expectations\']")
425
426     def test_tokenize_extra_equal(self):
427         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ = Qux.')
428         self.assertEqual(expectation.is_malformed(), True)
429         self.assertEqual(str(expectation.errors), '["Extraneous \'=\'"]')
430
431     def test_tokenize_valid(self):
432         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ')
433         self.assertEqual(expectation.is_malformed(), False)
434         self.assertEqual(expectation.comment, None)
435         self.assertEqual(len(expectation.errors), 0)
436
437     def test_tokenize_valid_with_comment(self):
438         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ //Qux.')
439         self.assertEqual(expectation.is_malformed(), False)
440         self.assertEqual(expectation.comment, 'Qux.')
441         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
442         self.assertEqual(str(expectation.expectations), '[\'baz\']')
443         self.assertEqual(len(expectation.errors), 0)
444
445     def test_tokenize_valid_with_multiple_modifiers(self):
446         expectation = TestExpectationParser.tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
447         self.assertEqual(expectation.is_malformed(), False)
448         self.assertEqual(expectation.comment, 'Qux.')
449         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
450         self.assertEqual(str(expectation.expectations), '[\'baz\']')
451         self.assertEqual(len(expectation.errors), 0)
452
453     def test_parse_empty_string(self):
454         test_port = port.get('test-win-xp', None)
455         test_port.test_exists = lambda test: True
456         test_config = test_port.test_configuration()
457         full_test_list = []
458         expectation_line = TestExpectationParser.tokenize('')
459         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
460         parser.parse(expectation_line)
461         self.assertFalse(expectation_line.is_invalid())
462
463
464 class TestExpectationSerializerTests(unittest.TestCase):
465     def __init__(self, testFunc):
466         test_port = port.get('test-win-xp', None)
467         self._serializer = TestExpectationSerializer(TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros()))
468         unittest.TestCase.__init__(self, testFunc)
469
470     def assert_round_trip(self, in_string, expected_string=None):
471         expectation = TestExpectationParser.tokenize(in_string)
472         if expected_string is None:
473             expected_string = in_string
474         self.assertEqual(expected_string, self._serializer.to_string(expectation))
475
476     def assert_list_round_trip(self, in_string, expected_string=None):
477         expectations = TestExpectationParser.tokenize_list(in_string)
478         if expected_string is None:
479             expected_string = in_string
480         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, SpecifierSorter()))
481
482     def test_unparsed_to_string(self):
483         expectation = TestExpectationLine()
484
485         self.assertEqual(self._serializer.to_string(expectation), '')
486         expectation.comment = 'Qux.'
487         self.assertEqual(self._serializer.to_string(expectation), '//Qux.')
488         expectation.name = 'bar'
489         self.assertEqual(self._serializer.to_string(expectation), ' : bar =  //Qux.')
490         expectation.modifiers = ['foo']
491         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar =  //Qux.')
492         expectation.expectations = ['bAz']
493         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
494         expectation.expectations = ['bAz1', 'baZ2']
495         self.assertEqual(self._serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
496         expectation.modifiers = ['foo1', 'foO2']
497         self.assertEqual(self._serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
498         expectation.errors.append('Oh the horror.')
499         self.assertEqual(self._serializer.to_string(expectation), '')
500         expectation.original_string = 'Yes it is!'
501         self.assertEqual(self._serializer.to_string(expectation), 'Yes it is!')
502
503     def test_parsed_to_string(self):
504         expectation_line = TestExpectationLine()
505         expectation_line.parsed_bug_modifiers = ['BUGX']
506         expectation_line.name = 'test/name/for/realz.html'
507         expectation_line.parsed_expectations = set([IMAGE])
508         self.assertEqual(self._serializer.to_string(expectation_line), None)
509         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')])
510         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE')
511         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')])
512         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
513         expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu')])
514         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE\nBUGX XP DEBUG GPU : test/name/for/realz.html = IMAGE')
515
516     def test_parsed_expectations_string(self):
517         expectation_line = TestExpectationLine()
518         expectation_line.parsed_expectations = set([])
519         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
520         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
521         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
522         expectation_line.parsed_expectations = set([PASS, FAIL])
523         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
524         expectation_line.parsed_expectations = set([FAIL, PASS])
525         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
526
527     def test_parsed_modifier_string(self):
528         expectation_line = TestExpectationLine()
529         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
530         expectation_line.parsed_modifiers = ['for', 'the']
531         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
532         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
533         expectation_line.parsed_bug_modifiers = []
534         expectation_line.parsed_modifiers = []
535         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
536         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
537         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
538         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
539         expectation_line.parsed_bug_modifiers = []
540         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
541         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
542
543
544     def test_format_result(self):
545         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
546         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
547
548     def test_string_roundtrip(self):
549         self.assert_round_trip('')
550         self.assert_round_trip('FOO')
551         self.assert_round_trip(':')
552         self.assert_round_trip('FOO :')
553         self.assert_round_trip('FOO : bar')
554         self.assert_round_trip('  FOO :')
555         self.assert_round_trip('    FOO : bar')
556         self.assert_round_trip('FOO : bar = BAZ')
557         self.assert_round_trip('FOO : bar = BAZ //Qux.')
558         self.assert_round_trip('FOO : bar = BAZ // Qux.')
559         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
560         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
561         self.assert_round_trip('FOO : : bar = BAZ')
562         self.assert_round_trip('FOO : : bar = BAZ')
563         self.assert_round_trip('FOO : : bar ==== BAZ')
564         self.assert_round_trip('=')
565         self.assert_round_trip('//')
566         self.assert_round_trip('// ')
567         self.assert_round_trip('// Foo')
568         self.assert_round_trip('// Foo')
569         self.assert_round_trip('// Foo :')
570         self.assert_round_trip('// Foo : =')
571
572     def test_list_roundtrip(self):
573         self.assert_list_round_trip('')
574         self.assert_list_round_trip('\n')
575         self.assert_list_round_trip('\n\n')
576         self.assert_list_round_trip('bar')
577         self.assert_list_round_trip('bar\n//Qux.')
578         self.assert_list_round_trip('bar\n//Qux.\n')
579
580     def test_string_whitespace_stripping(self):
581         self.assert_round_trip('\n', '')
582         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
583         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
584         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
585         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
586         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
587         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
588
589
590 class TestExpectationEditorTests(unittest.TestCase):
591     WIN_RELEASE_CPU_CONFIGS = set([
592         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
593         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
594         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
595     ])
596
597     RELEASE_CONFIGS = set([
598         TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
599         TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
600         TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
601         TestConfiguration(None, 'vista', 'x86', 'release', 'gpu'),
602         TestConfiguration(None, 'win7', 'x86', 'release', 'gpu'),
603         TestConfiguration(None, 'xp', 'x86', 'release', 'gpu'),
604         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'cpu'),
605         TestConfiguration(None, 'leopard', 'x86', 'release', 'cpu'),
606         TestConfiguration(None, 'snowleopard', 'x86', 'release', 'gpu'),
607         TestConfiguration(None, 'leopard', 'x86', 'release', 'gpu'),
608         TestConfiguration(None, 'lucid', 'x86', 'release', 'cpu'),
609         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'cpu'),
610         TestConfiguration(None, 'lucid', 'x86', 'release', 'gpu'),
611         TestConfiguration(None, 'lucid', 'x86_64', 'release', 'gpu'),
612     ])
613
614     def __init__(self, testFunc):
615         self.test_port = port.get('test-win-xp', None)
616         self.full_test_list = ['failures/expected/keyboard.html', 'failures/expected/audio.html']
617         unittest.TestCase.__init__(self, testFunc)
618
619     def make_parsed_expectation_lines(self, in_string):
620         expectation_lines = TestExpectationParser.tokenize_list(in_string)
621         parser = TestExpectationParser(self.test_port, self.full_test_list, allow_rebaseline_modifier=False)
622         for expectation_line in expectation_lines:
623             self.assertFalse(expectation_line.is_invalid())
624             parser.parse(expectation_line)
625         return expectation_lines
626
627     def assert_remove_roundtrip(self, in_string, test, expected_string, remove_flakes=False):
628         test_config_set = set([self.test_port.test_configuration()])
629         expectation_lines = self.make_parsed_expectation_lines(in_string)
630         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
631         editor.remove_expectation(test, test_config_set, remove_flakes)
632         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
633         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
634         self.assertEquals(result, expected_string)
635
636     def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, remove_flakes=False, parsed_bug_modifiers=None, test_configs=None):
637         test_config_set = test_configs or set([self.test_port.test_configuration()])
638         expectation_lines = self.make_parsed_expectation_lines(in_string)
639         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
640         editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifiers=parsed_bug_modifiers)
641         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
642         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
643         self.assertEquals(result, expected_string)
644
645     def test_remove_expectation(self):
646         self.assert_remove_roundtrip("""
647 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
648 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/hang.html', """
649 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
650 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
651
652         self.assert_remove_roundtrip("""
653 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
654 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
655 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
656 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
657
658         self.assert_remove_roundtrip("""
659 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
660 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
661 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
662 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
663
664         self.assert_remove_roundtrip("""
665 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
666 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html',  """
667 BUGX1 MAC : failures/expected/keyboard.html = IMAGE
668 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
669
670         self.assert_remove_roundtrip("""
671 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
672 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
673 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
674
675         self.assert_remove_roundtrip("""
676 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
677 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
678 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
679 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
680 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
681 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
682
683         self.assert_remove_roundtrip("""
684 BUGX1 XP : failures/expected/keyboard.html = IMAGE
685 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
686 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
687 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
688 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
689
690         self.assert_remove_roundtrip("""
691 BUGX1 : failures/expected/keyboard.html = IMAGE
692 BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
693 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
694 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
695 BUGX1 LINUX MAC VISTA WIN7 : failures/expected/keyboard.html = IMAGE
696 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
697
698         self.assert_remove_roundtrip("""
699 BUGX1 WIN : failures/expected = PASS
700 BUGX2 XP RELEASE : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
701 BUGX1 WIN : failures/expected = PASS
702 BUGX2 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE""")
703
704         self.assert_remove_roundtrip("""
705 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
706 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
707 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
708
709         self.assert_remove_roundtrip("""
710 BUGX1 WIN : failures/expected = FAIL""", 'failures/expected/keyboard.html', """
711 BUGX1 WIN : failures/expected = FAIL""")
712
713         self.assert_remove_roundtrip("""
714 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
715 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
716 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = PASS IMAGE
717 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
718
719         self.assert_remove_roundtrip("""
720 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
721 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
722 BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", remove_flakes=True)
723
724     def test_remove_expectation_multiple(self):
725         in_string = """
726 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
727 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
728         expectation_lines = self.make_parsed_expectation_lines(in_string)
729         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
730         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
731         test = "failures/expected/keyboard.html"
732
733         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]))
734         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
735 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
736 BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
737 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
738 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
739
740         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]))
741         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
742 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
743 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
744 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
745
746         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'win7', 'x86', 'release', 'gpu')]))
747         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
748 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
749 BUGX1 WIN7 DEBUG GPU : failures/expected/keyboard.html = IMAGE
750 BUGX1 WIN7 CPU : failures/expected/keyboard.html = IMAGE
751 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
752 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
753 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
754
755         editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')]))
756         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
757 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
758 BUGX1 WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
759 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
760 BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
761 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
762
763         editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'cpu'), TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'vista', 'x86', 'release', 'gpu')]))
764         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
765 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
766 BUGX1 VISTA WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
767 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
768
769         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
770         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
771 BUGX2 WIN : failures/expected/audio.html = IMAGE""")
772
773         test = "failures/expected/audio.html"
774
775         editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
776         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), "")
777
778     def test_update_expectation(self):
779         self.assert_update_roundtrip("""
780 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
781 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
782
783         self.assert_update_roundtrip("""
784 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '')
785
786         self.assert_update_roundtrip("""
787 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
788 BUGX1 XP RELEASE CPU : failures/expected = TEXT
789 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
790
791         self.assert_update_roundtrip("""
792 BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
793 BUGX1 XP RELEASE CPU : failures/expected = TEXT
794 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = PASS""")
795
796         self.assert_update_roundtrip("""
797 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([TEXT]), """
798 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""")
799
800         self.assert_update_roundtrip("""
801 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
802 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifiers=['BUGAWESOME'])
803
804         self.assert_update_roundtrip("""
805 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
806 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
807 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
808
809         self.assert_update_roundtrip("""
810 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
811 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT""")
812
813         self.assert_update_roundtrip("""
814 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
815 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
816 BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifiers=['BUGAWESOME'])
817
818         self.assert_update_roundtrip("""
819 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
820 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
821 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
822 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
823 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
824
825         self.assert_update_roundtrip("""
826 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
827 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
828 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
829 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT""")
830
831         self.assert_update_roundtrip("""
832 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
833 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
834 BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
835 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
836 BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
837
838         self.assert_update_roundtrip("""
839 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
840 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
841
842         self.assert_update_roundtrip("""
843 BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '', test_configs=self.WIN_RELEASE_CPU_CONFIGS)
844
845         self.assert_update_roundtrip("""
846 BUGX1 RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
847 BUGX1 LINUX MAC RELEASE CPU : failures/expected/keyboard.html = TEXT
848 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
849
850         self.assert_update_roundtrip("""
851 BUGX1 MAC : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
852 BUGX1 MAC : failures/expected/keyboard.html = TEXT
853 BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
854
855     def test_update_expectation_multiple(self):
856         in_string = """
857 BUGX1 WIN : failures/expected/keyboard.html = IMAGE
858 BUGX2 WIN : failures/expected/audio.html = IMAGE"""
859         expectation_lines = self.make_parsed_expectation_lines(in_string)
860         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
861         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
862         test = "failures/expected/keyboard.html"
863
864         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), ['BUG_UPDATE1'])
865         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
866 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
867 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
868 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
869 BUGX2 WIN : failures/expected/audio.html = IMAGE
870 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT""")
871
872         editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), ['BUG_UPDATE2'])
873         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
874 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
875 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
876 BUGX2 WIN : failures/expected/audio.html = IMAGE
877 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT
878 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT""")
879
880         editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), ['BUG_UPDATE3'])
881         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
882 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
883 BUGX1 WIN7 RELEASE GPU : failures/expected/keyboard.html = IMAGE
884 BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
885 BUGX1 VISTA XP GPU : failures/expected/keyboard.html = IMAGE
886 BUGX2 WIN : failures/expected/audio.html = IMAGE
887 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
888 BUG_UPDATE3 WIN RELEASE CPU : failures/expected/keyboard.html = CRASH""")
889
890         editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), ['BUG_UPDATE4'])
891         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
892 BUGX1 XP DEBUG GPU : failures/expected/keyboard.html = IMAGE
893 BUGX1 VISTA WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
894 BUGX2 WIN : failures/expected/audio.html = IMAGE
895 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
896 BUG_UPDATE4 RELEASE : failures/expected/keyboard.html = FAIL""")
897
898         editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), ['BUG_UPDATE5'])
899         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
900 BUGX2 WIN : failures/expected/audio.html = IMAGE
901 BUG_UPDATE5 : failures/expected/keyboard.html = TIMEOUT""")
902
903
904 if __name__ == '__main__':
905     unittest.main()