ews-build should clearly indicate flaky test failures
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 15 Apr 2019 23:36:25 +0000 (23:36 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 15 Apr 2019 23:36:25 +0000 (23:36 +0000)
https://bugs.webkit.org/show_bug.cgi?id=196947

Reviewed by Lucas Forschler.

* BuildSlaveSupport/ews-build/steps.py:
(AnalyzeAPITestsResults.analyzeResults):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@244306 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/ews-build/steps.py
Tools/ChangeLog

index 19bf3d8..2fb797b 100644 (file)
@@ -792,13 +792,17 @@ class AnalyzeAPITestsResults(buildstep.BuildStep):
         second_run_failures = getAPITestFailures(second_run_results)
         clean_tree_failures = getAPITestFailures(clean_tree_results)
 
-        self._addToLog('stderr', '\nFailures in API Test first run: {}'.format(first_run_failures))
-        self._addToLog('stderr', '\nFailures in API Test second run: {}'.format(first_run_failures))
-        self._addToLog('stderr', '\nFailures in API Test on clean tree: {}'.format(clean_tree_failures))
         failures_with_patch = first_run_failures.intersection(second_run_failures)
+        flaky_failures = first_run_failures.union(second_run_failures) - first_run_failures.intersection(second_run_failures)
+        flaky_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in flaky_failures])
         new_failures = failures_with_patch - clean_tree_failures
         new_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in new_failures])
 
+        self._addToLog('stderr', '\nFailures in API Test first run: {}'.format(first_run_failures))
+        self._addToLog('stderr', '\nFailures in API Test second run: {}'.format(second_run_failures))
+        self._addToLog('stderr', '\nFlaky Tests: {}'.format(flaky_failures))
+        self._addToLog('stderr', '\nFailures in API Test on clean tree: {}'.format(clean_tree_failures))
+
         if new_failures:
             self._addToLog('stderr', '\nNew failures: {}\n'.format(new_failures))
             self.finished(FAILURE)
@@ -812,6 +816,8 @@ class AnalyzeAPITestsResults(buildstep.BuildStep):
             self.build.results = SUCCESS
             self.descriptionDone = 'Passed API tests'
             message = 'Found {} pre-existing API tests failures'.format(len(clean_tree_failures))
+            if flaky_failures:
+                message += '. Flaky tests: {}'.format(flaky_failures_string)
             self.build.buildFinished([message], SUCCESS)
 
     @defer.inlineCallbacks
index 2382b22..ab9ffde 100644 (file)
@@ -1,3 +1,13 @@
+2019-04-15  Aakash Jain  <aakash_jain@apple.com>
+
+        ews-build should clearly indicate flaky test failures
+        https://bugs.webkit.org/show_bug.cgi?id=196947
+
+        Reviewed by Lucas Forschler.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (AnalyzeAPITestsResults.analyzeResults):
+
 2019-04-15  Brent Fulgham  <bfulgham@apple.com>
 
         InjectedBundle parameters often need initialization function called before unarchiving