Allow multiple bug identifiers in test expectations.
authordglazkov@chromium.org <dglazkov@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 3 Aug 2011 22:34:28 +0000 (22:34 +0000)
committerdglazkov@chromium.org <dglazkov@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 3 Aug 2011 22:34:28 +0000 (22:34 +0000)
https://bugs.webkit.org/show_bug.cgi?id=65642

Reviewed by Adam Barth.

* Scripts/webkitpy/layout_tests/models/test_expectations.py: Tweaked to allow multiple bug ids.
* Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py: Added tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@92317 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

index e3f9035..c901444 100644 (file)
@@ -1,5 +1,15 @@
 2011-08-03  Dimitri Glazkov  <dglazkov@chromium.org>
 
+        Allow multiple bug identifiers in test expectations.
+        https://bugs.webkit.org/show_bug.cgi?id=65642
+
+        Reviewed by Adam Barth.
+
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py: Tweaked to allow multiple bug ids.
+        * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py: Added tests.
+
+2011-08-03  Dimitri Glazkov  <dglazkov@chromium.org>
+
         Teach BuilderToPort to distinguish between Release and Debug builders.
         https://bugs.webkit.org/show_bug.cgi?id=65628
 
index cc39f9c..f8ef0e7 100644 (file)
@@ -132,7 +132,7 @@ class TestExpectationSerializer(object):
         if expectation_line.name is None:
             return '' if expectation_line.comment is None else "//%s" % expectation_line.comment
 
-        if expectation_line.parsed_bug_modifier:
+        if expectation_line.parsed_bug_modifiers:
             specifiers_list = self._test_configuration_converter.to_specifiers_list(expectation_line.matching_configurations)
             result = []
             for specifiers in specifiers_list:
@@ -152,9 +152,9 @@ class TestExpectationSerializer(object):
 
     def _parsed_modifier_string(self, expectation_line, specifiers):
         result = []
-        if expectation_line.parsed_bug_modifier:
-            result.append(expectation_line.parsed_bug_modifier)
-        result.extend(expectation_line.parsed_modifiers)
+        if expectation_line.parsed_bug_modifiers:
+            result.extend(sorted(expectation_line.parsed_bug_modifiers))
+        result.extend(sorted(expectation_line.parsed_modifiers))
         result.extend(self._test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
         return ' '.join(result)
 
@@ -215,11 +215,11 @@ class TestExpectationParser(object):
                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
                     expectation_line.errors.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
                 else:
-                    expectation_line.parsed_bug_modifier = modifier
+                    expectation_line.parsed_bug_modifiers.append(modifier)
             else:
                 parsed_specifiers.add(modifier)
 
-        if not expectation_line.parsed_bug_modifier and not has_wontfix:
+        if not expectation_line.parsed_bug_modifiers and not has_wontfix:
             expectation_line.warnings.append('Test lacks BUG modifier.')
 
         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers:
@@ -344,7 +344,7 @@ class TestExpectationLine:
         self.path = None
         self.modifiers = []
         self.parsed_modifiers = []
-        self.parsed_bug_modifier = None
+        self.parsed_bug_modifiers = []
         self.matching_configurations = set()
         self.expectations = []
         self.parsed_expectations = set()
@@ -581,7 +581,7 @@ class TestExpectationsModel(object):
 
 class BugManager(object):
     """A simple interface for managing bugs from TestExpectationsEditor."""
-    def close_bug(self, bug_id, reference_bug_id=None):
+    def close_bug(self, bug_ids, reference_bug_ids=None):
         raise NotImplementedError("BugManager.close_bug")
 
     def create_bug(self):
@@ -631,31 +631,31 @@ class TestExpectationsEditor(object):
             if (not expectation_line.is_flaky() or remove_flakes) and expectation_line.matching_configurations & test_config_set:
                 expectation_line.matching_configurations = expectation_line.matching_configurations - test_config_set
                 if not expectation_line.matching_configurations:
-                    self._bug_manager.close_bug(expectation_line.parsed_bug_modifier)
+                    self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers)
                 return
 
-    def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifier=None):
+    def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifiers=None):
         """Updates expectations for {test} in the set of test configuration {test_config_set} to the values of {expectation_set}.
-        If {parsed_bug_modifier} is supplied, it is used for updated expectations. Otherwise, a new bug is created.
+        If {parsed_bug_modifiers} is supplied, it is used for updated expectations. Otherwise, a new bug is created.
 
         Here, we treat updating expectations to PASS as special: if possible, the corresponding lines are completely removed.
         """
         # FIXME: Allow specifying modifiers (SLOW, SKIP, WONTFIX).
         expectation_lines = self._test_to_expectation_lines.get(test, [])
         remaining_configurations = test_config_set.copy()
-        bug_id = self._get_valid_bug_id(parsed_bug_modifier)
+        bug_ids = self._get_valid_bug_ids(parsed_bug_modifiers)
         remove_expectations = expectation_set == set([PASS]) and test not in self._tests_with_directory_paths
         for expectation_line in expectation_lines:
             if expectation_line.matching_configurations == remaining_configurations:
                 # Tweak expectations on existing line.
                 if expectation_line.parsed_expectations == expectation_set:
                     return
-                self._bug_manager.close_bug(expectation_line.parsed_bug_modifier, bug_id)
+                self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
                 if remove_expectations:
                     expectation_line.matching_configurations = set()
                     return
                 expectation_line.parsed_expectations = expectation_set
-                expectation_line.parsed_bug_modifier = bug_id
+                expectation_line.parsed_bug_modifiers = bug_ids
                 return
             elif expectation_line.matching_configurations >= remaining_configurations:
                 # 1) Split up into two expectation lines:
@@ -666,7 +666,7 @@ class TestExpectationsEditor(object):
                 break
             elif expectation_line.matching_configurations <= remaining_configurations:
                 # Remove existing expectation line.
-                self._bug_manager.close_bug(expectation_line.parsed_bug_modifier, bug_id)
+                self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
                 expectation_line.matching_configurations = set()
             else:
                 intersection = expectation_line.matching_configurations & remaining_configurations
@@ -674,16 +674,16 @@ class TestExpectationsEditor(object):
                     expectation_line.matching_configurations -= intersection
 
         if not remove_expectations:
-            self._expectation_lines.append(self._create_new_line(test, bug_id, remaining_configurations, expectation_set))
+            self._expectation_lines.append(self._create_new_line(test, bug_ids, remaining_configurations, expectation_set))
 
-    def _get_valid_bug_id(self, suggested_bug_id):
+    def _get_valid_bug_ids(self, suggested_bug_ids):
         # FIXME: Flesh out creating a bug properly (title, etc.)
-        return suggested_bug_id or self._bug_manager.create_bug()
+        return suggested_bug_ids or [self._bug_manager.create_bug()]
 
-    def _create_new_line(self, name, bug_id, config_set, expectation_set):
+    def _create_new_line(self, name, bug_ids, config_set, expectation_set):
         new_line = TestExpectationLine()
         new_line.name = name
-        new_line.parsed_bug_modifier = bug_id
+        new_line.parsed_bug_modifiers = bug_ids
         new_line.matching_configurations = config_set
         new_line.parsed_expectations = expectation_set
         # Ensure index integrity for multiple operations.
index f7f2941..444b16d 100644 (file)
@@ -502,7 +502,7 @@ class TestExpectationSerializerTests(unittest.TestCase):
 
     def test_parsed_to_string(self):
         expectation_line = TestExpectationLine()
-        expectation_line.parsed_bug_modifier = 'BUGX'
+        expectation_line.parsed_bug_modifiers = ['BUGX']
         expectation_line.name = 'test/name/for/realz.html'
         expectation_line.parsed_expectations = set([IMAGE])
         self.assertEqual(self._serializer.to_string(expectation_line), None)
@@ -526,14 +526,20 @@ class TestExpectationSerializerTests(unittest.TestCase):
 
     def test_parsed_modifier_string(self):
         expectation_line = TestExpectationLine()
-        expectation_line.parsed_bug_modifier = 'garden-o-matic'
+        expectation_line.parsed_bug_modifiers = ['garden-o-matic']
         expectation_line.parsed_modifiers = ['for', 'the']
         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
-        expectation_line.parsed_bug_modifier = ''
+        expectation_line.parsed_bug_modifiers = []
         expectation_line.parsed_modifiers = []
         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
+        expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
+        self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
+        expectation_line.parsed_bug_modifiers = []
+        expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
+        self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
+
 
     def test_format_result(self):
         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
@@ -627,11 +633,11 @@ class TestExpectationEditorTests(unittest.TestCase):
         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
         self.assertEquals(result, expected_string)
 
-    def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, remove_flakes=False, parsed_bug_modifier=None, test_configs=None):
+    def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, remove_flakes=False, parsed_bug_modifiers=None, test_configs=None):
         test_config_set = test_configs or set([self.test_port.test_configuration()])
         expectation_lines = self.make_parsed_expectation_lines(in_string)
         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
-        editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifier=parsed_bug_modifier)
+        editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifiers=parsed_bug_modifiers)
         converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
         result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
         self.assertEquals(result, expected_string)
@@ -793,7 +799,7 @@ BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""")
 
         self.assert_update_roundtrip("""
 BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
-BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
+BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifiers=['BUGAWESOME'])
 
         self.assert_update_roundtrip("""
 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
@@ -807,7 +813,7 @@ BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT""")
         self.assert_update_roundtrip("""
 BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
 BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
-BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
+BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifiers=['BUGAWESOME'])
 
         self.assert_update_roundtrip("""
 BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
@@ -855,7 +861,7 @@ BUGX2 WIN : failures/expected/audio.html = IMAGE"""
         editor = TestExpectationsEditor(expectation_lines, MockBugManager())
         test = "failures/expected/keyboard.html"
 
-        editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), 'BUG_UPDATE1')
+        editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), ['BUG_UPDATE1'])
         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
 BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
@@ -863,7 +869,7 @@ BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
 BUGX2 WIN : failures/expected/audio.html = IMAGE
 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT""")
 
-        editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), 'BUG_UPDATE2')
+        editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), ['BUG_UPDATE2'])
         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
 BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
 BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
@@ -871,7 +877,7 @@ BUGX2 WIN : failures/expected/audio.html = IMAGE
 BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT
 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT""")
 
-        editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), 'BUG_UPDATE3')
+        editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), ['BUG_UPDATE3'])
         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
 BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
 BUGX1 WIN7 RELEASE GPU : failures/expected/keyboard.html = IMAGE
@@ -881,7 +887,7 @@ BUGX2 WIN : failures/expected/audio.html = IMAGE
 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
 BUG_UPDATE3 WIN RELEASE CPU : failures/expected/keyboard.html = CRASH""")
 
-        editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), 'BUG_UPDATE4')
+        editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), ['BUG_UPDATE4'])
         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
 BUGX1 XP DEBUG GPU : failures/expected/keyboard.html = IMAGE
 BUGX1 VISTA WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
@@ -889,7 +895,7 @@ BUGX2 WIN : failures/expected/audio.html = IMAGE
 BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
 BUG_UPDATE4 RELEASE : failures/expected/keyboard.html = FAIL""")
 
-        editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), 'BUG_UPDATE5')
+        editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), ['BUG_UPDATE5'])
         self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
 BUGX2 WIN : failures/expected/audio.html = IMAGE
 BUG_UPDATE5 : failures/expected/keyboard.html = TIMEOUT""")