webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
index 40f9cb5..2046446 100644 (file)
@@ -1,4 +1,5 @@
 # Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2013 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -28,6 +29,8 @@
 
 import unittest
 
+from collections import OrderedDict
+
 from webkitpy.common.host_mock import MockHost
 from webkitpy.common.system.outputcapture import OutputCapture
 
@@ -35,12 +38,6 @@ from webkitpy.layout_tests.models.test_configuration import *
 from webkitpy.layout_tests.models.test_expectations import *
 from webkitpy.layout_tests.models.test_configuration import *
 
-try:
-    from collections import OrderedDict
-except ImportError:
-    # Needed for Python < 2.7
-    from webkitpy.thirdparty.ordered_dict import OrderedDict
-
 
 class Base(unittest.TestCase):
     # Note that all of these tests are written assuming the configuration
@@ -52,25 +49,29 @@ class Base(unittest.TestCase):
         self._exp = None
         unittest.TestCase.__init__(self, testFunc)
 
-    def get_test(self, test_name):
-        # FIXME: Remove this routine and just reference test names directly.
-        return test_name
-
     def get_basic_tests(self):
-        return [self.get_test('failures/expected/text.html'),
-                self.get_test('failures/expected/image_checksum.html'),
-                self.get_test('failures/expected/crash.html'),
-                self.get_test('failures/expected/missing_text.html'),
-                self.get_test('failures/expected/image.html'),
-                self.get_test('passes/text.html')]
+        return ['failures/expected/text.html',
+                'failures/expected/image_checksum.html',
+                'failures/expected/crash.html',
+                'failures/expected/leak.html',
+                'failures/expected/flaky-leak.html',
+                'failures/expected/missing_text.html',
+                'failures/expected/image.html',
+                'failures/expected/reftest.html',
+                'failures/expected/leaky-reftest.html',
+                'passes/text.html']
 
     def get_basic_expectations(self):
         return """
 Bug(test) failures/expected/text.html [ Failure ]
 Bug(test) failures/expected/crash.html [ WontFix ]
+Bug(test) failures/expected/leak.html [ Leak ]
+Bug(test) failures/expected/flaky-leak.html [ Failure Leak ]
 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
 Bug(test) failures/expected/image_checksum.html [ WontFix ]
 Bug(test) failures/expected/image.html [ WontFix Mac ]
+Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
+Bug(test) failures/expected/leaky-reftest.html [ ImageOnlyFailure Leak ]
 """
 
     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
@@ -78,13 +79,16 @@ Bug(test) failures/expected/image.html [ WontFix Mac ]
         expectations_dict['expectations'] = expectations
         if overrides:
             expectations_dict['overrides'] = overrides
-        self._port.expectations_dict = lambda: expectations_dict
+        self._port.expectations_dict = lambda **kwargs: expectations_dict
         expectations_to_lint = expectations_dict if is_lint_mode else None
         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
+        self._exp.parse_all_expectations()
 
     def assert_exp(self, test, result):
-        self.assertEqual(self._exp.get_expectations(self.get_test(test)),
-                          set([result]))
+        self.assertEqual(self._exp.model().get_expectations(test), set([result]))
+
+    def assert_exp_set(self, test, result_set):
+        self.assertEqual(self._exp.model().get_expectations(test), result_set)
 
     def assert_bad_expectations(self, expectations, overrides=None):
         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
@@ -95,15 +99,20 @@ class BasicTests(Base):
         self.parse_exp(self.get_basic_expectations())
         self.assert_exp('failures/expected/text.html', FAIL)
         self.assert_exp('failures/expected/image_checksum.html', PASS)
+        self.assert_exp('failures/expected/reftest.html', IMAGE)
+        self.assert_exp_set('failures/expected/leaky-reftest.html', set([IMAGE, LEAK]))
+        self.assert_exp('failures/expected/leak.html', LEAK)
+        self.assert_exp_set('failures/expected/flaky-leak.html', set([FAIL, LEAK]))
         self.assert_exp('passes/text.html', PASS)
+        # self.assert_exp_set('passes/flaky-leak.html', set([PASS, LEAK]))
         self.assert_exp('failures/expected/image.html', PASS)
 
 
 class MiscTests(Base):
     def test_multiple_results(self):
         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
-        self.assertEqual(self._exp.get_expectations(
-            self.get_test('failures/expected/text.html')),
+        self.assertEqual(self._exp.model().get_expectations(
+            'failures/expected/text.html'),
             set([FAIL, CRASH]))
 
     def test_result_was_expected(self):
@@ -126,6 +135,14 @@ class MiscTests(Base):
         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
 
+    def test_remove_leak_failures(self):
+        self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL])), set([FAIL]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([LEAK])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, LEAK])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL, LEAK])), set([FAIL]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, IMAGE, LEAK, CRASH])), set([PASS, IMAGE, CRASH]))
+
     def test_suffixes_for_expectations(self):
         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
@@ -139,46 +156,41 @@ class MiscTests(Base):
         exp_str = 'Bug(x) failures/expected [ WontFix ]'
         self.parse_exp(exp_str)
         test_name = 'failures/expected/unknown-test.html'
-        unknown_test = self.get_test(test_name)
-        self.assertRaises(KeyError, self._exp.get_expectations,
+        unknown_test = test_name
+        self.assertRaises(KeyError, self._exp.model().get_expectations,
                           unknown_test)
         self.assert_exp('failures/expected/crash.html', PASS)
 
     def test_get_modifiers(self):
         self.parse_exp(self.get_basic_expectations())
-        self.assertEqual(self._exp.get_modifiers(
-                         self.get_test('passes/text.html')), [])
+        self.assertEqual(self._exp.model().get_modifiers('passes/text.html'), [])
 
     def test_get_expectations_string(self):
         self.parse_exp(self.get_basic_expectations())
-        self.assertEqual(self._exp.get_expectations_string(
-                          self.get_test('failures/expected/text.html')),
-                          'FAIL')
+        self.assertEqual(self._exp.model().get_expectations_string('failures/expected/text.html'), 'FAIL')
 
     def test_expectation_to_string(self):
         # Normal cases are handled by other tests.
         self.parse_exp(self.get_basic_expectations())
-        self.assertRaises(ValueError, self._exp.expectation_to_string,
-                          -1)
+        self.assertRaises(ValueError, self._exp.model().expectation_to_string, -1)
 
     def test_get_test_set(self):
         # Handle some corner cases for this routine not covered by other tests.
         self.parse_exp(self.get_basic_expectations())
-        s = self._exp.get_test_set(WONTFIX)
+        s = self._exp.model().get_test_set(WONTFIX)
         self.assertEqual(s,
-            set([self.get_test('failures/expected/crash.html'),
-                 self.get_test('failures/expected/image_checksum.html')]))
+            set(['failures/expected/crash.html',
+                 'failures/expected/image_checksum.html']))
 
     def test_parse_warning(self):
         try:
             filesystem = self._port.host.filesystem
             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
-            self.get_test('disabled-test.html-disabled'),
             self.parse_exp("[ FOO ] failures/expected/text.html [ Failure ]\n"
                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
             self.assertFalse(True, "ParseError wasn't raised")
-        except ParseError, e:
+        except ParseError as e:
             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
                         "expectations:2 Path does not exist. non-existent-test.html")
             self.assertEqual(str(e), warnings)
@@ -222,20 +234,46 @@ class MiscTests(Base):
 
     def test_pixel_tests_flag(self):
         def match(test, result, pixel_tests_enabled):
-            return self._exp.matches_an_expected_result(
-                self.get_test(test), result, pixel_tests_enabled)
+            expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, False)
+            return self._exp.matches_an_expected_result(test, result, expectations)
+
+        self.parse_exp(self.get_basic_expectations())
+        pixel_tests_enabled = True
+        pixel_tests_disabled = False
+        self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_enabled))
+        self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_disabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_disabled))
+        self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_enabled))
+        self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_disabled))
+        self.assertTrue(match('failures/expected/crash.html', PASS, pixel_tests_disabled))
+        self.assertTrue(match('passes/text.html', PASS, pixel_tests_disabled))
+
+    def test_world_leaks_flag(self):
+        def match(test, result, pixel_tests_enabled, world_leaks_enabled):
+            expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, world_leaks_enabled)
+            return self._exp.matches_an_expected_result(test, result, expectations)
+
+        pixel_tests_enabled = True
+        pixel_tests_disabled = False
+        world_leaks_enabled = True
+        world_leaks_disabled = False
 
         self.parse_exp(self.get_basic_expectations())
-        self.assertTrue(match('failures/expected/text.html', FAIL, True))
-        self.assertTrue(match('failures/expected/text.html', FAIL, False))
-        self.assertFalse(match('failures/expected/text.html', CRASH, True))
-        self.assertFalse(match('failures/expected/text.html', CRASH, False))
-        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
-                              True))
-        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
-                              False))
-        self.assertTrue(match('failures/expected/crash.html', PASS, False))
-        self.assertTrue(match('passes/text.html', PASS, False))
+        self.assertTrue(match('failures/expected/leak.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leak.html', PASS, pixel_tests_enabled, world_leaks_disabled))
+        self.assertTrue(match('failures/expected/flaky-leak.html', FAIL, pixel_tests_enabled, world_leaks_disabled))
+
+        self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_disabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', PASS, pixel_tests_disabled, world_leaks_disabled))
+
+        self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_disabled))
+
+        self.assertFalse(match('failures/expected/text.html', PASS, pixel_tests_enabled, world_leaks_enabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled, world_leaks_disabled))
+        self.assertTrue(match('passes/text.html', PASS, pixel_tests_enabled, world_leaks_disabled))
 
     def test_more_specific_override_resets_skip(self):
         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
@@ -243,26 +281,27 @@ class MiscTests(Base):
         self.assert_exp('failures/expected/text.html', IMAGE)
         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
                                                      'failures/expected/text.html') in
-                         self._exp.get_tests_with_result_type(SKIP))
+                         self._exp.model().get_tests_with_result_type(SKIP))
 
 
 class SkippedTests(Base):
     def check(self, expectations, overrides, skips, lint=False):
-        port = MockHost().port_factory.get('qt')
+        port = MockHost().port_factory.get('mac')
         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
         expectations_dict = OrderedDict()
         expectations_dict['expectations'] = expectations
         if overrides:
             expectations_dict['overrides'] = overrides
-        port.expectations_dict = lambda: expectations_dict
-        port.skipped_layout_tests = lambda tests: set(skips)
+        port.expectations_dict = lambda **kwargs: expectations_dict
+        port.skipped_layout_tests = lambda tests, **kwargs: set(skips)
         expectations_to_lint = expectations_dict if lint else None
         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
+        exp.parse_all_expectations()
 
         # Check that the expectation is for BUG_DUMMY SKIP : ... [ Pass ]
-        self.assertEqual(exp.get_modifiers('failures/expected/text.html'),
+        self.assertEqual(exp.model().get_modifiers('failures/expected/text.html'),
                           [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER])
-        self.assertEqual(exp.get_expectations('failures/expected/text.html'), set([PASS]))
+        self.assertEqual(exp.model().get_expectations('failures/expected/text.html'), set([PASS]))
 
     def test_skipped_tests_work(self):
         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'])
@@ -287,14 +326,15 @@ class SkippedTests(Base):
                    skips=['failures/expected'])
 
     def test_skipped_entry_dont_exist(self):
-        port = MockHost().port_factory.get('qt')
+        port = MockHost().port_factory.get('mac')
         expectations_dict = OrderedDict()
         expectations_dict['expectations'] = ''
-        port.expectations_dict = lambda: expectations_dict
-        port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
+        port.expectations_dict = lambda **kwargs: expectations_dict
+        port.skipped_layout_tests = lambda tests, **kwargs: set(['foo/bar/baz.html'])
         capture = OutputCapture()
         capture.capture_output()
         exp = TestExpectations(port)
+        exp.parse_all_expectations()
         _, _, logs = capture.restore_output()
         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
 
@@ -329,9 +369,8 @@ class ExpectationSyntaxTests(Base):
 
     def test_bare_name_and_bugs(self):
         self.assert_tokenize_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
-        self.assert_tokenize_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
         self.assert_tokenize_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
-        self.assert_tokenize_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
+        self.assert_tokenize_exp('webkit.org/b/12345 webkit.org/b/34567 foo.html', modifiers=['BUGWK12345', 'BUGWK34567', 'SKIP'], expectations=['PASS'])
 
     def test_comments(self):
         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
@@ -346,7 +385,7 @@ class ExpectationSyntaxTests(Base):
         self.assert_tokenize_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
 
     def test_unknown_expectation(self):
-        self.assert_tokenize_exp('foo.html [ Audio ]', expectations=['Audio'])
+        self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
 
     def test_skip(self):
         self.assert_tokenize_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
@@ -354,8 +393,13 @@ class ExpectationSyntaxTests(Base):
     def test_slow(self):
         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
 
+    def test_leak(self):
+        self.assert_tokenize_exp('foo.html [ Leak ]', modifiers=[], expectations=['LEAK'])
+
     def test_wontfix(self):
         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
+        self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
+        self.assert_tokenize_exp('foo.html [ WontFix Pass Failure ]', modifiers=['WONTFIX'], expectations=['PASS', 'FAIL'])
 
     def test_blank_line(self):
         self.assert_tokenize_exp('', name=None)
@@ -363,9 +407,9 @@ class ExpectationSyntaxTests(Base):
     def test_warnings(self):
         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
         self.assert_tokenize_exp('[ [', warnings=['unexpected "["'], name=None)
-        self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
+        self.assert_tokenize_exp('webkit.org/b/12345 ]', warnings=['unexpected "]"'], name=None)
 
-        self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
+        self.assert_tokenize_exp('foo.html webkit.org/b/12345 ]', warnings=['"webkit.org/b/12345" is not at the start of the line.'])
 
 
 class SemanticTests(Base):
@@ -376,7 +420,7 @@ class SemanticTests(Base):
         try:
             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
             self.fail('should have raised an error about a bad bug identifier')
-        except ParseError, exp:
+        except ParseError as exp:
             self.assertEqual(len(exp.warnings), 1)
 
     def test_missing_bugid(self):
@@ -391,16 +435,16 @@ class SemanticTests(Base):
         self.assertEqual(line.warnings, ['Test lacks BUG modifier.'])
 
     def test_skip_and_wontfix(self):
-        # Skip and WontFix are not allowed to have other expectations as well, because those
+        # Skip is not allowed to have other expectations as well, because those
         # expectations won't be exercised and may become stale .
         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
         self.assertTrue(self._exp.has_warnings())
 
         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
-        self.assertTrue(self._exp.has_warnings())
+        self.assertFalse(self._exp.has_warnings())
 
         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
-        self.assertTrue(self._exp.has_warnings())
+        self.assertFalse(self._exp.has_warnings())
 
     def test_slow_and_timeout(self):
         # A test cannot be SLOW and expected to TIMEOUT.
@@ -473,14 +517,15 @@ class RemoveConfigurationsTest(Base):
         test_port.test_isfile = lambda test: True
 
         test_config = test_port.test_configuration()
-        test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
+        test_port.expectations_dict = lambda **kwargs: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 """}
         expectations = TestExpectations(test_port, self.get_basic_tests())
+        expectations.parse_all_expectations()
 
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 
-        self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
+        self.assertEqual("""Bug(x) [ 7SP0 Linux Vista Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 """, actual_expectations)
 
@@ -491,14 +536,15 @@ Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
         test_port.test_isfile = lambda test: True
 
         test_config = test_port.test_configuration()
-        test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
+        test_port.expectations_dict = lambda **kwargs: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 """}
         expectations = TestExpectations(test_port)
+        expectations.parse_all_expectations()
 
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
-        actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
+        actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-7sp0', None).test_configuration())
 
         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 """, actual_expectations)
@@ -695,7 +741,8 @@ class TestExpectationSerializationTests(unittest.TestCase):
         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
 
-    def test_string_whitespace_stripping(self):
+    def disabled_test_string_whitespace_stripping(self):
+        # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
         self.assert_round_trip('\n', '')
         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')