2010-08-26 Dirk Pranke <dpranke@chromium.org>

        Reviewed by Tony Chang.

        new-run-webkit-tests: add rebaselining tests for test_expectations

        This patch adds more unit tests for the rebaselining code in the
        test_expectations module. It doesn't add any tests for
        rebaseline_chromium_webkit_tests itself; that will come some other
        time.

        https://bugs.webkit.org/show_bug.cgi?id=44648

        * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
        * Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@66125 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog
index bbc2693813c3f3a7338ebacf6f4966f51cdceadb..03378b724c01ad5a323731cee295b51e8e53202d 100644 (file)
--- a/WebKitTools/ChangeLog
+++ b/WebKitTools/ChangeLog
@@ -1,3 +1,19 @@
+2010-08-26  Dirk Pranke  <dpranke@chromium.org>
+
+        Reviewed by Tony Chang.
+
+        new-run-webkit-tests: add rebaselining tests for test_expectations
+        
+        This patch adds more unit tests for the rebaselining code in the
+        test_expectations module. It doesn't add any tests for
+        rebaseline_chromium_webkit_tests itself; that will come some other
+        time.
+
+        https://bugs.webkit.org/show_bug.cgi?id=44648
+
+        * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
+        * Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py:
+
 2010-08-26  Gustavo Noronha Silva  <gustavo.noronha@collabora.co.uk>
 
         Reviewed by Martin Robinson.

diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
index 086321d707d59f5c1aaa5731e3a611c493e6de49..3d8349b302a1f5a7d656108a07f4eb31a90996e4 100644 (file)
--- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
@@ -477,6 +477,7 @@ class TestExpectationsFile:
           the updated string.
         """
 
+        assert(platform)
         f_orig = self._get_iterable_expectations(self._expectations)
         f_new = []
 
@@ -487,6 +488,8 @@ class TestExpectationsFile:
             lineno += 1
             action = self._get_platform_update_action(line, lineno, tests,
                                                       platform)
+            assert(action in (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM,
+                              ADD_PLATFORMS_EXCEPT_THIS))
             if action == NO_CHANGE:
                 # Save the original line back to the file
                 _log.debug('No change to test: %s', line)
@@ -522,9 +525,6 @@ class TestExpectationsFile:
                 _log.info('Test updated: ')
                 _log.info('  old: %s', line)
                 _log.info('  new: %s', new_line)
-            else:
-                _log.error('Unknown update action: %d; line: %s',
-                           action, line)
 
         _log.info('Total tests removed: %d', tests_removed)
         _log.info('Total tests updated: %d', tests_updated)

diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
index 22214b054bc8bfc48041dc1333457f9dbf4e9dfb..e4c6b7e8450f343aff40e77a7266299ab5fa86ae 100644 (file)
--- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
@@ -93,14 +93,16 @@ class Base(unittest.TestCase):
                 self.get_test('failures/expected/image_checksum.html'),
                 self.get_test('failures/expected/crash.html'),
                 self.get_test('failures/expected/missing_text.html'),
+                self.get_test('failures/expected/image.html'),
                 self.get_test('passes/text.html')]
 
     def get_basic_expectations(self):
         return """
 BUG_TEST : failures/expected/text.html = TEXT
-BUG_TEST SKIP : failures/expected/crash.html = CRASH
+BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
 BUG_TEST REBASELINE : failure/expected/missing_image.html = MISSING
-BUG_TEST : failures/expected/image_checksum.html = IMAGE
+BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
+BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
 """
 
     def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
@@ -125,6 +127,13 @@ class TestExpectationsTest(Base):
         self.assert_exp('failures/expected/text.html', TEXT)
         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
         self.assert_exp('passes/text.html', PASS)
+        self.assert_exp('failures/expected/image.html', PASS)
+
+    def test_multiple_results(self):
+        self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
+        self.assertEqual(self._exp.get_expectations(
+            self.get_test('failures/expected/text.html')),
+            set([TEXT, CRASH]))
 
     def test_defer(self):
         self.parse_exp('BUGX DEFER : failures/expected/text.html = TEXT')
@@ -183,6 +192,20 @@ BUGX DEFER : failures/expected = IMAGE
         self.assertRaises(ValueError, self._exp.expectation_to_string,
                           -1)
 
+    def test_get_test_set(self):
+        # Handle some corner cases for this routine not covered by other tests.
+        self.parse_exp(self.get_basic_expectations())
+        s = self._exp._expected_failures.get_test_set(WONTFIX)
+        self.assertEqual(s,
+            set([self.get_test('failures/expected/crash.html'),
+                 self.get_test('failures/expected/image_checksum.html')]))
+        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
+        self.assertEqual(s,
+            set([self.get_test('failures/expected/crash.html')]))
+        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
+                                                      include_skips=False)
+        self.assertEqual(s, set([]))
+
     def test_syntax_missing_expectation(self):
         # This is missing the expectation.
         self.assertRaises(SyntaxError, self.parse_exp,
@@ -256,20 +279,43 @@ BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
 
 class RebaseliningTest(Base):
     """Test rebaselining-specific functionality."""
+    def assertRemove(self, platform, input_expectations, expected_expectations):
+        self.parse_exp(input_expectations)
+        test = self.get_test('failures/expected/text.html')
+        actual_expectations = self._exp.remove_platform_from_expectations(
+            test, platform)
+        self.assertEqual(expected_expectations, actual_expectations)
+
     def test_no_get_rebaselining_failures(self):
         self.parse_exp(self.get_basic_expectations())
         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 
-    def test_basic(self):
+    def test_get_rebaselining_failures_expand(self):
         self.parse_exp("""
 BUG_TEST REBASELINE : failures/expected/text.html = TEXT
 """)
         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
 
-        new_exp_str = self._exp.remove_platform_from_expectations(
-            self.get_test('failures/expected/text.html'), 'TEST')
-        # FIXME: actually test rebaselining
-        # self.assertEqual(new_exp_str, '\n')
+    def test_remove_expand(self):
+        self.assertRemove('mac',
+            'BUGX REBASELINE : failures/expected/text.html = TEXT\n',
+            'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
+
+    def test_remove_mac_win(self):
+        self.assertRemove('mac',
+            'BUGX REBASELINE MAC WIN : failures/expected/text.html = TEXT\n',
+            'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
+
+    def test_remove_mac_mac(self):
+        self.assertRemove('mac',
+            'BUGX REBASELINE MAC : failures/expected/text.html = TEXT\n',
+            '')
+
+    def test_remove_nothing(self):
+        self.assertRemove('mac',
+            '\n\n',
+            '\n\n')
+
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py
index 4fe3ec16cf515e17ac78577c10ff3af405f1ab04..d36b540c75aa8f7d049612a043e0247e2998a07f 100644 (file)
--- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py
@@ -118,10 +118,10 @@ class TestPort(base.Port):
             return file.read()
 
     def test_base_platform_names(self):
-        return ('test',)
+        return ('mac', 'win')
 
     def test_platform_name(self):
-        return 'test'
+        return 'mac'
 
     def test_platform_names(self):
         return self.test_base_platform_names()

diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py
index c0bb4ace189400093d7a212d1f42b20eb2a21470..747b8b402f7b21436ffb167058817d693e5b45e9 100644 (file)
--- a/WebKitTools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py
@@ -93,7 +93,7 @@ class TestExpectationsTestCase(unittest.TestCase):
             ["passes/text.html = CRASH TIMEOUT FAIL PASS"],
             "")
         self.assert_lines_lint(
-            ["BUG1234 TEST : passes/text.html = PASS FAIL"],
+            ["BUG1234 MAC : passes/text.html = PASS FAIL"],
             "")
         self.assert_lines_lint(
             ["SKIP BUG1234 : passes/text.html = TIMEOUT PASS"],
@@ -105,10 +105,10 @@ class TestExpectationsTestCase(unittest.TestCase):
             ["BUG1234 DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
             "")
         self.assert_lines_lint(
-            ["BUG1234 TEST DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
+            ["BUG1234 MAC DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
             "")
         self.assert_lines_lint(
-            ["BUG1234 DEBUG TEST : passes/text.html = TIMEOUT PASS"],
+            ["BUG1234 DEBUG MAC : passes/text.html = TIMEOUT PASS"],
             "")
         self.assert_lines_lint(
             ["SLOW DEFER BUG1234 : passes/text.html = PASS"],