Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 # Copyright (C) 2013 Apple Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import unittest
  31
  32 from collections import OrderedDict
  33
  34 from webkitpy.common.host_mock import MockHost
  35 from webkitpy.common.system.outputcapture import OutputCapture
  36
  37 from webkitpy.layout_tests.models.test_configuration import *
  38 from webkitpy.layout_tests.models.test_expectations import *
  39 from webkitpy.layout_tests.models.test_configuration import *
  40
  41
  42 class Base(unittest.TestCase):
  43     # Note that all of these tests are written assuming the configuration
  44     # being tested is Windows XP, Release build.
  45
  46     def __init__(self, testFunc):
  47         host = MockHost()
  48         self._port = host.port_factory.get('test-win-xp', None)
  49         self._exp = None
  50         unittest.TestCase.__init__(self, testFunc)
  51
  52     def get_basic_tests(self):
  53         return ['failures/expected/text.html',
  54                 'failures/expected/image_checksum.html',
  55                 'failures/expected/crash.html',
  56                 'failures/expected/leak.html',
  57                 'failures/expected/flaky-leak.html',
  58                 'failures/expected/missing_text.html',
  59                 'failures/expected/image.html',
  60                 'failures/expected/reftest.html',
  61                 'failures/expected/leaky-reftest.html',
  62                 'passes/text.html']
  63
  64     def get_basic_expectations(self):
  65         return """
  66 Bug(test) failures/expected/text.html [ Failure ]
  67 Bug(test) failures/expected/crash.html [ WontFix ]
  68 Bug(test) failures/expected/leak.html [ Leak ]
  69 Bug(test) failures/expected/flaky-leak.html [ Failure Leak ]
  70 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
  71 Bug(test) failures/expected/image_checksum.html [ WontFix ]
  72 Bug(test) failures/expected/image.html [ WontFix Mac ]
  73 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
  74 Bug(test) failures/expected/leaky-reftest.html [ ImageOnlyFailure Leak ]
  75 """
  76
  77     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
  78         expectations_dict = OrderedDict()
  79         expectations_dict['expectations'] = expectations
  80         if overrides:
  81             expectations_dict['overrides'] = overrides
  82         self._port.expectations_dict = lambda: expectations_dict
  83         expectations_to_lint = expectations_dict if is_lint_mode else None
  84         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
  85         self._exp.parse_all_expectations()
  86
  87     def assert_exp(self, test, result):
  88         self.assertEqual(self._exp.model().get_expectations(test), set([result]))
  89
  90     def assert_exp_set(self, test, result_set):
  91         self.assertEqual(self._exp.model().get_expectations(test), result_set)
  92
  93     def assert_bad_expectations(self, expectations, overrides=None):
  94         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
  95
  96
  97 class BasicTests(Base):
  98     def test_basic(self):
  99         self.parse_exp(self.get_basic_expectations())
 100         self.assert_exp('failures/expected/text.html', FAIL)
 101         self.assert_exp('failures/expected/image_checksum.html', PASS)
 102         self.assert_exp('failures/expected/reftest.html', IMAGE)
 103         self.assert_exp_set('failures/expected/leaky-reftest.html', set([IMAGE, LEAK]))
 104         self.assert_exp('failures/expected/leak.html', LEAK)
 105         self.assert_exp_set('failures/expected/flaky-leak.html', set([FAIL, LEAK]))
 106         self.assert_exp('passes/text.html', PASS)
 107         # self.assert_exp_set('passes/flaky-leak.html', set([PASS, LEAK]))
 108         self.assert_exp('failures/expected/image.html', PASS)
 109
 110
 111 class MiscTests(Base):
 112     def test_multiple_results(self):
 113         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
 114         self.assertEqual(self._exp.model().get_expectations(
 115             'failures/expected/text.html'),
 116             set([FAIL, CRASH]))
 117
 118     def test_result_was_expected(self):
 119         # test basics
 120         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True)
 121         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
 122
 123         # test handling of SKIPped tests and results
 124         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True)
 125         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False)
 126
 127         # test handling of MISSING results and the REBASELINE modifier
 128         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True)
 129         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False)
 130
 131     def test_remove_pixel_failures(self):
 132         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 133         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
 134         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
 135         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 136         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
 137
 138     def test_remove_leak_failures(self):
 139         self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL])), set([FAIL]))
 140         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS])), set([PASS]))
 141         self.assertEqual(TestExpectations.remove_leak_failures(set([LEAK])), set([PASS]))
 142         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, LEAK])), set([PASS]))
 143         self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL, LEAK])), set([FAIL]))
 144         self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, IMAGE, LEAK, CRASH])), set([PASS, IMAGE, CRASH]))
 145
 146     def test_suffixes_for_expectations(self):
 147         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
 148         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
 149         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
 150         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
 151
 152     def test_category_expectations(self):
 153         # This test checks unknown tests are not present in the
 154         # expectations and that known test part of a test category is
 155         # present in the expectations.
 156         exp_str = 'Bug(x) failures/expected [ WontFix ]'
 157         self.parse_exp(exp_str)
 158         test_name = 'failures/expected/unknown-test.html'
 159         unknown_test = test_name
 160         self.assertRaises(KeyError, self._exp.model().get_expectations,
 161                           unknown_test)
 162         self.assert_exp('failures/expected/crash.html', PASS)
 163
 164     def test_get_modifiers(self):
 165         self.parse_exp(self.get_basic_expectations())
 166         self.assertEqual(self._exp.model().get_modifiers('passes/text.html'), [])
 167
 168     def test_get_expectations_string(self):
 169         self.parse_exp(self.get_basic_expectations())
 170         self.assertEqual(self._exp.model().get_expectations_string('failures/expected/text.html'), 'FAIL')
 171
 172     def test_expectation_to_string(self):
 173         # Normal cases are handled by other tests.
 174         self.parse_exp(self.get_basic_expectations())
 175         self.assertRaises(ValueError, self._exp.model().expectation_to_string, -1)
 176
 177     def test_get_test_set(self):
 178         # Handle some corner cases for this routine not covered by other tests.
 179         self.parse_exp(self.get_basic_expectations())
 180         s = self._exp.model().get_test_set(WONTFIX)
 181         self.assertEqual(s,
 182             set(['failures/expected/crash.html',
 183                  'failures/expected/image_checksum.html']))
 184
 185     def test_parse_warning(self):
 186         try:
 187             filesystem = self._port.host.filesystem
 188             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
 189             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
 190                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
 191                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
 192             self.assertFalse(True, "ParseError wasn't raised")
 193         except ParseError as e:
 194             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
 195                         "expectations:2 Path does not exist. non-existent-test.html")
 196             self.assertEqual(str(e), warnings)
 197
 198     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
 199         oc = OutputCapture()
 200         try:
 201             oc.capture_output()
 202             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
 203         finally:
 204             _, _, logs = oc.restore_output()
 205             self.assertNotEquals(logs, '')
 206
 207     def test_error_on_different_platform(self):
 208         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
 209         self.assertRaises(ParseError, self.parse_exp,
 210             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
 211             is_lint_mode=True)
 212
 213     def test_error_on_different_build_type(self):
 214         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
 215         self.assertRaises(ParseError, self.parse_exp,
 216             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
 217             is_lint_mode=True)
 218
 219     def test_overrides(self):
 220         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 221                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
 222         self.assert_exp('failures/expected/text.html', IMAGE)
 223
 224     def test_overrides__directory(self):
 225         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 226                        "Bug(override) failures/expected [ Crash ]")
 227         self.assert_exp('failures/expected/text.html', CRASH)
 228         self.assert_exp('failures/expected/image.html', CRASH)
 229
 230     def test_overrides__duplicate(self):
 231         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
 232                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
 233                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
 234
 235     def test_pixel_tests_flag(self):
 236         def match(test, result, pixel_tests_enabled):
 237             expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, False)
 238             return self._exp.matches_an_expected_result(test, result, expectations)
 239
 240         self.parse_exp(self.get_basic_expectations())
 241         pixel_tests_enabled = True
 242         pixel_tests_disabled = False
 243         self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_enabled))
 244         self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_disabled))
 245         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled))
 246         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_disabled))
 247         self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_enabled))
 248         self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_disabled))
 249         self.assertTrue(match('failures/expected/crash.html', PASS, pixel_tests_disabled))
 250         self.assertTrue(match('passes/text.html', PASS, pixel_tests_disabled))
 251
 252     def test_world_leaks_flag(self):
 253         def match(test, result, pixel_tests_enabled, world_leaks_enabled):
 254             expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, world_leaks_enabled)
 255             return self._exp.matches_an_expected_result(test, result, expectations)
 256
 257         pixel_tests_enabled = True
 258         pixel_tests_disabled = False
 259         world_leaks_enabled = True
 260         world_leaks_disabled = False
 261
 262         self.parse_exp(self.get_basic_expectations())
 263         self.assertTrue(match('failures/expected/leak.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
 264         self.assertTrue(match('failures/expected/leak.html', PASS, pixel_tests_enabled, world_leaks_disabled))
 265         self.assertTrue(match('failures/expected/flaky-leak.html', FAIL, pixel_tests_enabled, world_leaks_disabled))
 266
 267         self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_disabled, world_leaks_enabled))
 268         self.assertTrue(match('failures/expected/leaky-reftest.html', PASS, pixel_tests_disabled, world_leaks_disabled))
 269
 270         self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_enabled))
 271         self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
 272         self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_disabled))
 273
 274         self.assertFalse(match('failures/expected/text.html', PASS, pixel_tests_enabled, world_leaks_enabled))
 275         self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled, world_leaks_disabled))
 276         self.assertTrue(match('passes/text.html', PASS, pixel_tests_enabled, world_leaks_disabled))
 277
 278     def test_more_specific_override_resets_skip(self):
 279         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
 280                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
 281         self.assert_exp('failures/expected/text.html', IMAGE)
 282         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
 283                                                      'failures/expected/text.html') in
 284                          self._exp.model().get_tests_with_result_type(SKIP))
 285
 286
 287 class SkippedTests(Base):
 288     def check(self, expectations, overrides, skips, lint=False):
 289         port = MockHost().port_factory.get('mac')
 290         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
 291         expectations_dict = OrderedDict()
 292         expectations_dict['expectations'] = expectations
 293         if overrides:
 294             expectations_dict['overrides'] = overrides
 295         port.expectations_dict = lambda: expectations_dict
 296         port.skipped_layout_tests = lambda tests: set(skips)
 297         expectations_to_lint = expectations_dict if lint else None
 298         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
 299         exp.parse_all_expectations()
 300
 301         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
 302         self.assertEqual(exp.model().get_modifiers('failures/expected/text.html'),
 303                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
 304         self.assertEqual(exp.model().get_expectations('failures/expected/text.html'), set([PASS]))
 305
 306     def test_skipped_tests_work(self):
 307         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
 308
 309     def test_duplicate_skipped_test_fails_lint(self):
 310         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, skips=['failures/expected/text.html'], lint=True)
 311
 312     def test_skipped_file_overrides_expectations(self):
 313         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 314                    skips=['failures/expected/text.html'])
 315
 316     def test_skipped_dir_overrides_expectations(self):
 317         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 318                    skips=['failures/expected'])
 319
 320     def test_skipped_file_overrides_overrides(self):
 321         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 322                    skips=['failures/expected/text.html'])
 323
 324     def test_skipped_dir_overrides_overrides(self):
 325         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 326                    skips=['failures/expected'])
 327
 328     def test_skipped_entry_dont_exist(self):
 329         port = MockHost().port_factory.get('mac')
 330         expectations_dict = OrderedDict()
 331         expectations_dict['expectations'] = ''
 332         port.expectations_dict = lambda: expectations_dict
 333         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
 334         capture = OutputCapture()
 335         capture.capture_output()
 336         exp = TestExpectations(port)
 337         exp.parse_all_expectations()
 338         _, _, logs = capture.restore_output()
 339         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
 340
 341
 342 class ExpectationSyntaxTests(Base):
 343     def test_unrecognized_expectation(self):
 344         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
 345
 346     def test_macro(self):
 347         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
 348         self.parse_exp(exp_str)
 349         self.assert_exp('failures/expected/text.html', FAIL)
 350
 351     def assert_tokenize_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
 352         bugs = bugs or []
 353         modifiers = modifiers or []
 354         expectations = expectations or []
 355         warnings = warnings or []
 356         filename = 'TestExpectations'
 357         line_number = 1
 358         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
 359         self.assertEqual(expectation_line.warnings, warnings)
 360         self.assertEqual(expectation_line.name, name)
 361         self.assertEqual(expectation_line.filename, filename)
 362         self.assertEqual(expectation_line.line_number, line_number)
 363         if not warnings:
 364             self.assertEqual(expectation_line.modifiers, modifiers)
 365             self.assertEqual(expectation_line.expectations, expectations)
 366
 367     def test_bare_name(self):
 368         self.assert_tokenize_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
 369
 370     def test_bare_name_and_bugs(self):
 371         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
 372         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
 373         self.assert_tokenize_exp('webkit.org/b/12345 webkit.org/b/34567 foo.html', modifiers=['BUGWK12345', 'BUGWK34567', 'SKIP'], expectations=['PASS'])
 374
 375     def test_comments(self):
 376         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
 377         self.assert_tokenize_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
 378
 379     def test_config_modifiers(self):
 380         self.assert_tokenize_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
 381         self.assert_tokenize_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
 382         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
 383
 384     def test_unknown_config(self):
 385         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
 386
 387     def test_unknown_expectation(self):
 388         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
 389
 390     def test_skip(self):
 391         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
 392
 393     def test_slow(self):
 394         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
 395
 396     def test_leak(self):
 397         self.assert_tokenize_exp('foo.html [ Leak ]', modifiers=[], expectations=['LEAK'])
 398
 399     def test_wontfix(self):
 400         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
 401         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
 402         self.assert_tokenize_exp('foo.html [ WontFix Pass Failure ]', modifiers=['WONTFIX'], expectations=['PASS', 'FAIL'])
 403
 404     def test_blank_line(self):
 405         self.assert_tokenize_exp('', name=None)
 406
 407     def test_warnings(self):
 408         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
 409         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
 410         self.assert_tokenize_exp('webkit.org/b/12345 ]', warnings=['unexpected "]"'], name=None)
 411
 412         self.assert_tokenize_exp('foo.html webkit.org/b/12345 ]', warnings=['"webkit.org/b/12345" is not at the start of the line.'])
 413
 414
 415 class SemanticTests(Base):
 416     def test_bug_format(self):
 417         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 418
 419     def test_bad_bugid(self):
 420         try:
 421             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 422             self.fail('should have raised an error about a bad bug identifier')
 423         except ParseError as exp:
 424             self.assertEqual(len(exp.warnings), 1)
 425
 426     def test_missing_bugid(self):
 427         self.parse_exp('failures/expected/text.html [ Failure ]')
 428         self.assertFalse(self._exp.has_warnings())
 429
 430         self._port.warn_if_bug_missing_in_test_expectations = lambda: True
 431
 432         self.parse_exp('failures/expected/text.html [ Failure ]')
 433         line = self._exp._model.get_expectation_line('failures/expected/text.html')
 434         self.assertFalse(line.is_invalid())
 435         self.assertEqual(line.warnings, ['Test lacks BUG modifier.'])
 436
 437     def test_skip_and_wontfix(self):
 438         # Skip is not allowed to have other expectations as well, because those
 439         # expectations won't be exercised and may become stale .
 440         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
 441         self.assertTrue(self._exp.has_warnings())
 442
 443         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
 444         self.assertFalse(self._exp.has_warnings())
 445
 446         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
 447         self.assertFalse(self._exp.has_warnings())
 448
 449     def test_slow_and_timeout(self):
 450         # A test cannot be SLOW and expected to TIMEOUT.
 451         self.assertRaises(ParseError, self.parse_exp,
 452             'Bug(test) failures/expected/timeout.html [ Slow Timeout ]', is_lint_mode=True)
 453
 454     def test_rebaseline(self):
 455         # Can't lint a file w/ 'REBASELINE' in it.
 456         self.assertRaises(ParseError, self.parse_exp,
 457             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
 458             is_lint_mode=True)
 459
 460     def test_duplicates(self):
 461         self.assertRaises(ParseError, self.parse_exp, """
 462 Bug(exp) failures/expected/text.html [ Failure ]
 463 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 464
 465         self.assertRaises(ParseError, self.parse_exp,
 466             self.get_basic_expectations(), overrides="""
 467 Bug(override) failures/expected/text.html [ Failure ]
 468 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 469
 470     def test_missing_file(self):
 471         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
 472         self.assertTrue(self._exp.has_warnings(), 1)
 473
 474
 475 class PrecedenceTests(Base):
 476     def test_file_over_directory(self):
 477         # This tests handling precedence of specific lines over directories
 478         # and tests expectations covering entire directories.
 479         exp_str = """
 480 Bug(x) failures/expected/text.html [ Failure ]
 481 Bug(y) failures/expected [ WontFix ]
 482 """
 483         self.parse_exp(exp_str)
 484         self.assert_exp('failures/expected/text.html', FAIL)
 485         self.assert_exp('failures/expected/crash.html', PASS)
 486
 487         exp_str = """
 488 Bug(x) failures/expected [ WontFix ]
 489 Bug(y) failures/expected/text.html [ Failure ]
 490 """
 491         self.parse_exp(exp_str)
 492         self.assert_exp('failures/expected/text.html', FAIL)
 493         self.assert_exp('failures/expected/crash.html', PASS)
 494
 495     def test_ambiguous(self):
 496         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 497                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
 498
 499     def test_more_modifiers(self):
 500         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 501                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
 502
 503     def test_order_in_file(self):
 504         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
 505                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
 506
 507     def test_macro_overrides(self):
 508         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
 509                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
 510
 511
 512 class RemoveConfigurationsTest(Base):
 513     def test_remove(self):
 514         host = MockHost()
 515         test_port = host.port_factory.get('test-win-xp', None)
 516         test_port.test_exists = lambda test: True
 517         test_port.test_isfile = lambda test: True
 518
 519         test_config = test_port.test_configuration()
 520         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
 521 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 522 """}
 523         expectations = TestExpectations(test_port, self.get_basic_tests())
 524         expectations.parse_all_expectations()
 525
 526         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 527
 528         self.assertEqual("""Bug(x) [ 7SP0 Linux Vista Release ] failures/expected/foo.html [ Failure ]
 529 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 530 """, actual_expectations)
 531
 532     def test_remove_line(self):
 533         host = MockHost()
 534         test_port = host.port_factory.get('test-win-xp', None)
 535         test_port.test_exists = lambda test: True
 536         test_port.test_isfile = lambda test: True
 537
 538         test_config = test_port.test_configuration()
 539         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 540 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 541 """}
 542         expectations = TestExpectations(test_port)
 543         expectations.parse_all_expectations()
 544
 545         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 546         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
 547         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-7sp0', None).test_configuration())
 548
 549         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 550 """, actual_expectations)
 551
 552
 553 class RebaseliningTest(Base):
 554     """Test rebaselining-specific functionality."""
 555     def assertRemove(self, input_expectations, input_overrides, tests, expected_expectations, expected_overrides):
 556         self.parse_exp(input_expectations, is_lint_mode=False, overrides=input_overrides)
 557         actual_expectations = self._exp.remove_rebaselined_tests(tests, 'expectations')
 558         self.assertEqual(expected_expectations, actual_expectations)
 559         actual_overrides = self._exp.remove_rebaselined_tests(tests, 'overrides')
 560         self.assertEqual(expected_overrides, actual_overrides)
 561
 562     def test_remove(self):
 563         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
 564                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
 565                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
 566                           'Bug(x0) failures/expected/image.html [ Crash ]\n',
 567                           ['failures/expected/text.html'],
 568                           'Bug(y) failures/expected/image.html [ ImageOnlyFailure Rebaseline ]\n'
 569                           'Bug(z) failures/expected/crash.html [ Crash ]\n',
 570                           'Bug(x0) failures/expected/image.html [ Crash ]\n')
 571
 572         # Ensure that we don't modify unrelated lines, even if we could rewrite them.
 573         # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html"
 574         self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n'
 575                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
 576                           'Bug(z) failures/expected/crash.html\n',
 577                           '',
 578                           ['failures/expected/text.html'],
 579                           'Bug(Y) failures/expected/image.html [ Skip   ]\n'
 580                           'Bug(z) failures/expected/crash.html\n',
 581                           '')
 582
 583     def test_get_rebaselining_failures(self):
 584         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
 585         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
 586         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
 587
 588         self.parse_exp(self.get_basic_expectations())
 589         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 590
 591
 592 class TestExpectationSerializationTests(unittest.TestCase):
 593     def __init__(self, testFunc):
 594         host = MockHost()
 595         test_port = host.port_factory.get('test-win-xp', None)
 596         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 597         unittest.TestCase.__init__(self, testFunc)
 598
 599     def _tokenize(self, line):
 600         return TestExpectationParser._tokenize_line('path', line, 0)
 601
 602     def assert_round_trip(self, in_string, expected_string=None):
 603         expectation = self._tokenize(in_string)
 604         if expected_string is None:
 605             expected_string = in_string
 606         self.assertEqual(expected_string, expectation.to_string(self._converter))
 607
 608     def assert_list_round_trip(self, in_string, expected_string=None):
 609         host = MockHost()
 610         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], allow_rebaseline_modifier=False)
 611         expectations = parser.parse('path', in_string)
 612         if expected_string is None:
 613             expected_string = in_string
 614         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
 615
 616     def test_unparsed_to_string(self):
 617         expectation = TestExpectationLine()
 618
 619         self.assertEqual(expectation.to_string(self._converter), '')
 620         expectation.comment = ' Qux.'
 621         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
 622         expectation.name = 'bar'
 623         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
 624         expectation.modifiers = ['foo']
 625         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 626         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
 627         expectation.expectations = ['bAz']
 628         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
 629         expectation.expectations = ['bAz1', 'baZ2']
 630         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
 631         expectation.modifiers = ['foo1', 'foO2']
 632         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
 633         expectation.warnings.append('Oh the horror.')
 634         self.assertEqual(expectation.to_string(self._converter), '')
 635         expectation.original_string = 'Yes it is!'
 636         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
 637
 638     def test_unparsed_list_to_string(self):
 639         expectation = TestExpectationLine()
 640         expectation.comment = 'Qux.'
 641         expectation.name = 'bar'
 642         expectation.modifiers = ['foo']
 643         expectation.expectations = ['bAz1', 'baZ2']
 644         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 645         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
 646
 647     def test_parsed_to_string(self):
 648         expectation_line = TestExpectationLine()
 649         expectation_line.parsed_bug_modifiers = ['BUGX']
 650         expectation_line.name = 'test/name/for/realz.html'
 651         expectation_line.parsed_expectations = set([IMAGE])
 652         self.assertEqual(expectation_line.to_string(self._converter), None)
 653         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
 654         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
 655         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
 656         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
 657
 658     def test_serialize_parsed_expectations(self):
 659         expectation_line = TestExpectationLine()
 660         expectation_line.parsed_expectations = set([])
 661         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
 662         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
 663         expectation_line.parsed_expectations = set([FAIL])
 664         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
 665         expectation_line.parsed_expectations = set([PASS, IMAGE])
 666         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass image')
 667         expectation_line.parsed_expectations = set([FAIL, PASS])
 668         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
 669
 670     def test_serialize_parsed_modifier_string(self):
 671         expectation_line = TestExpectationLine()
 672         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
 673         expectation_line.parsed_modifiers = ['for', 'the']
 674         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), 'garden-o-matic for the')
 675         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic for the win')
 676         expectation_line.parsed_bug_modifiers = []
 677         expectation_line.parsed_modifiers = []
 678         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, []), '')
 679         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'win')
 680         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
 681         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
 682         expectation_line.parsed_bug_modifiers = []
 683         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
 684         self.assertEqual(expectation_line._serialize_parsed_modifiers(self._converter, ['win']), 'garden-o-matic is total win')
 685
 686     def test_format_line(self):
 687         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
 688         self.assertEqual(TestExpectationLine._format_line(['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
 689
 690     def test_string_roundtrip(self):
 691         self.assert_round_trip('')
 692         self.assert_round_trip('FOO')
 693         self.assert_round_trip('[')
 694         self.assert_round_trip('FOO [')
 695         self.assert_round_trip('FOO ] bar')
 696         self.assert_round_trip('  FOO [')
 697         self.assert_round_trip('    [ FOO ] ')
 698         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
 699         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 700         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 701         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
 702         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
 703         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
 704         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
 705         self.assert_round_trip('FOO ] ] bar ==== BAZ')
 706         self.assert_round_trip('=')
 707         self.assert_round_trip('#')
 708         self.assert_round_trip('# ')
 709         self.assert_round_trip('# Foo')
 710         self.assert_round_trip('# Foo')
 711         self.assert_round_trip('# Foo :')
 712         self.assert_round_trip('# Foo : =')
 713
 714     def test_list_roundtrip(self):
 715         self.assert_list_round_trip('')
 716         self.assert_list_round_trip('\n')
 717         self.assert_list_round_trip('\n\n')
 718         self.assert_list_round_trip('bar')
 719         self.assert_list_round_trip('bar\n# Qux.')
 720         self.assert_list_round_trip('bar\n# Qux.\n')
 721
 722     def test_reconstitute_only_these(self):
 723         lines = []
 724         reconstitute_only_these = []
 725
 726         def add_line(matching_configurations, reconstitute):
 727             expectation_line = TestExpectationLine()
 728             expectation_line.original_string = "Nay"
 729             expectation_line.parsed_bug_modifiers = ['BUGX']
 730             expectation_line.name = 'Yay'
 731             expectation_line.parsed_expectations = set([IMAGE])
 732             expectation_line.matching_configurations = matching_configurations
 733             lines.append(expectation_line)
 734             if reconstitute:
 735                 reconstitute_only_these.append(expectation_line)
 736
 737         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
 738         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
 739         serialized = TestExpectations.list_to_string(lines, self._converter)
 740         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
 741         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
 742         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
 743
 744     def disabled_test_string_whitespace_stripping(self):
 745         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
 746         self.assert_round_trip('\n', '')
 747         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 748         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 749         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 750         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 751         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 752         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')