webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations_unittest.py
index 639005b..2046446 100644 (file)
@@ -53,17 +53,25 @@ class Base(unittest.TestCase):
         return ['failures/expected/text.html',
                 'failures/expected/image_checksum.html',
                 'failures/expected/crash.html',
+                'failures/expected/leak.html',
+                'failures/expected/flaky-leak.html',
                 'failures/expected/missing_text.html',
                 'failures/expected/image.html',
+                'failures/expected/reftest.html',
+                'failures/expected/leaky-reftest.html',
                 'passes/text.html']
 
     def get_basic_expectations(self):
         return """
 Bug(test) failures/expected/text.html [ Failure ]
 Bug(test) failures/expected/crash.html [ WontFix ]
+Bug(test) failures/expected/leak.html [ Leak ]
+Bug(test) failures/expected/flaky-leak.html [ Failure Leak ]
 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
 Bug(test) failures/expected/image_checksum.html [ WontFix ]
 Bug(test) failures/expected/image.html [ WontFix Mac ]
+Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
+Bug(test) failures/expected/leaky-reftest.html [ ImageOnlyFailure Leak ]
 """
 
     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
@@ -71,14 +79,16 @@ Bug(test) failures/expected/image.html [ WontFix Mac ]
         expectations_dict['expectations'] = expectations
         if overrides:
             expectations_dict['overrides'] = overrides
-        self._port.expectations_dict = lambda: expectations_dict
+        self._port.expectations_dict = lambda **kwargs: expectations_dict
         expectations_to_lint = expectations_dict if is_lint_mode else None
         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_to_lint=expectations_to_lint)
         self._exp.parse_all_expectations()
 
     def assert_exp(self, test, result):
-        self.assertEqual(self._exp.model().get_expectations(test),
-                          set([result]))
+        self.assertEqual(self._exp.model().get_expectations(test), set([result]))
+
+    def assert_exp_set(self, test, result_set):
+        self.assertEqual(self._exp.model().get_expectations(test), result_set)
 
     def assert_bad_expectations(self, expectations, overrides=None):
         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
@@ -89,7 +99,12 @@ class BasicTests(Base):
         self.parse_exp(self.get_basic_expectations())
         self.assert_exp('failures/expected/text.html', FAIL)
         self.assert_exp('failures/expected/image_checksum.html', PASS)
+        self.assert_exp('failures/expected/reftest.html', IMAGE)
+        self.assert_exp_set('failures/expected/leaky-reftest.html', set([IMAGE, LEAK]))
+        self.assert_exp('failures/expected/leak.html', LEAK)
+        self.assert_exp_set('failures/expected/flaky-leak.html', set([FAIL, LEAK]))
         self.assert_exp('passes/text.html', PASS)
+        # self.assert_exp_set('passes/flaky-leak.html', set([PASS, LEAK]))
         self.assert_exp('failures/expected/image.html', PASS)
 
 
@@ -120,6 +135,14 @@ class MiscTests(Base):
         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
 
+    def test_remove_leak_failures(self):
+        self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL])), set([FAIL]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([LEAK])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, LEAK])), set([PASS]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([FAIL, LEAK])), set([FAIL]))
+        self.assertEqual(TestExpectations.remove_leak_failures(set([PASS, IMAGE, LEAK, CRASH])), set([PASS, IMAGE, CRASH]))
+
     def test_suffixes_for_expectations(self):
         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
@@ -149,8 +172,7 @@ class MiscTests(Base):
     def test_expectation_to_string(self):
         # Normal cases are handled by other tests.
         self.parse_exp(self.get_basic_expectations())
-        self.assertRaises(ValueError, self._exp.model().expectation_to_string,
-                          -1)
+        self.assertRaises(ValueError, self._exp.model().expectation_to_string, -1)
 
     def test_get_test_set(self):
         # Handle some corner cases for this routine not covered by other tests.
@@ -168,7 +190,7 @@ class MiscTests(Base):
                 "Bug(rniwa) non-existent-test.html [ Failure ]\n"
                 "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
             self.assertFalse(True, "ParseError wasn't raised")
-        except ParseError, e:
+        except ParseError as e:
             warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n"
                         "expectations:2 Path does not exist. non-existent-test.html")
             self.assertEqual(str(e), warnings)
@@ -212,20 +234,46 @@ class MiscTests(Base):
 
     def test_pixel_tests_flag(self):
         def match(test, result, pixel_tests_enabled):
-            return self._exp.matches_an_expected_result(
-                test, result, pixel_tests_enabled)
+            expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, False)
+            return self._exp.matches_an_expected_result(test, result, expectations)
+
+        self.parse_exp(self.get_basic_expectations())
+        pixel_tests_enabled = True
+        pixel_tests_disabled = False
+        self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_enabled))
+        self.assertTrue(match('failures/expected/text.html', FAIL, pixel_tests_disabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_disabled))
+        self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_enabled))
+        self.assertTrue(match('failures/expected/image_checksum.html', PASS, pixel_tests_disabled))
+        self.assertTrue(match('failures/expected/crash.html', PASS, pixel_tests_disabled))
+        self.assertTrue(match('passes/text.html', PASS, pixel_tests_disabled))
+
+    def test_world_leaks_flag(self):
+        def match(test, result, pixel_tests_enabled, world_leaks_enabled):
+            expectations = self._exp.filtered_expectations_for_test(test, pixel_tests_enabled, world_leaks_enabled)
+            return self._exp.matches_an_expected_result(test, result, expectations)
+
+        pixel_tests_enabled = True
+        pixel_tests_disabled = False
+        world_leaks_enabled = True
+        world_leaks_disabled = False
 
         self.parse_exp(self.get_basic_expectations())
-        self.assertTrue(match('failures/expected/text.html', FAIL, True))
-        self.assertTrue(match('failures/expected/text.html', FAIL, False))
-        self.assertFalse(match('failures/expected/text.html', CRASH, True))
-        self.assertFalse(match('failures/expected/text.html', CRASH, False))
-        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
-                              True))
-        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
-                              False))
-        self.assertTrue(match('failures/expected/crash.html', PASS, False))
-        self.assertTrue(match('passes/text.html', PASS, False))
+        self.assertTrue(match('failures/expected/leak.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leak.html', PASS, pixel_tests_enabled, world_leaks_disabled))
+        self.assertTrue(match('failures/expected/flaky-leak.html', FAIL, pixel_tests_enabled, world_leaks_disabled))
+
+        self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_disabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', PASS, pixel_tests_disabled, world_leaks_disabled))
+
+        self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', LEAK, pixel_tests_enabled, world_leaks_enabled))
+        self.assertTrue(match('failures/expected/leaky-reftest.html', IMAGE, pixel_tests_enabled, world_leaks_disabled))
+
+        self.assertFalse(match('failures/expected/text.html', PASS, pixel_tests_enabled, world_leaks_enabled))
+        self.assertFalse(match('failures/expected/text.html', CRASH, pixel_tests_enabled, world_leaks_disabled))
+        self.assertTrue(match('passes/text.html', PASS, pixel_tests_enabled, world_leaks_disabled))
 
     def test_more_specific_override_resets_skip(self):
         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
@@ -244,8 +292,8 @@ class SkippedTests(Base):
         expectations_dict['expectations'] = expectations
         if overrides:
             expectations_dict['overrides'] = overrides
-        port.expectations_dict = lambda: expectations_dict
-        port.skipped_layout_tests = lambda tests: set(skips)
+        port.expectations_dict = lambda **kwargs: expectations_dict
+        port.skipped_layout_tests = lambda tests, **kwargs: set(skips)
         expectations_to_lint = expectations_dict if lint else None
         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_to_lint=expectations_to_lint)
         exp.parse_all_expectations()
@@ -281,8 +329,8 @@ class SkippedTests(Base):
         port = MockHost().port_factory.get('mac')
         expectations_dict = OrderedDict()
         expectations_dict['expectations'] = ''
-        port.expectations_dict = lambda: expectations_dict
-        port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
+        port.expectations_dict = lambda **kwargs: expectations_dict
+        port.skipped_layout_tests = lambda tests, **kwargs: set(['foo/bar/baz.html'])
         capture = OutputCapture()
         capture.capture_output()
         exp = TestExpectations(port)
@@ -345,6 +393,9 @@ class ExpectationSyntaxTests(Base):
     def test_slow(self):
         self.assert_tokenize_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
 
+    def test_leak(self):
+        self.assert_tokenize_exp('foo.html [ Leak ]', modifiers=[], expectations=['LEAK'])
+
     def test_wontfix(self):
         self.assert_tokenize_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', modifiers=['WONTFIX'], expectations=['IMAGE'])
@@ -369,7 +420,7 @@ class SemanticTests(Base):
         try:
             self.parse_exp('BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
             self.fail('should have raised an error about a bad bug identifier')
-        except ParseError, exp:
+        except ParseError as exp:
             self.assertEqual(len(exp.warnings), 1)
 
     def test_missing_bugid(self):
@@ -466,7 +517,7 @@ class RemoveConfigurationsTest(Base):
         test_port.test_isfile = lambda test: True
 
         test_config = test_port.test_configuration()
-        test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
+        test_port.expectations_dict = lambda **kwargs: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 """}
         expectations = TestExpectations(test_port, self.get_basic_tests())
@@ -474,7 +525,7 @@ Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 
-        self.assertEqual("""Bug(x) [ Linux Vista Win7 Release ] failures/expected/foo.html [ Failure ]
+        self.assertEqual("""Bug(x) [ 7SP0 Linux Vista Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 """, actual_expectations)
 
@@ -485,7 +536,7 @@ Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
         test_port.test_isfile = lambda test: True
 
         test_config = test_port.test_configuration()
-        test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
+        test_port.expectations_dict = lambda **kwargs: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 """}
         expectations = TestExpectations(test_port)
@@ -493,7 +544,7 @@ Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
-        actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
+        actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-7sp0', None).test_configuration())
 
         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 """, actual_expectations)