[ews-build] Retry API test in case of failures
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 21 Mar 2019 23:11:58 +0000 (23:11 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 21 Mar 2019 23:11:58 +0000 (23:11 +0000)
https://bugs.webkit.org/show_bug.cgi?id=196004

Reviewed by Lucas Forschler.

* BuildSlaveSupport/ews-build/steps.py:
(UnApplyPatchIfRequired.doStepIf): Updated doStepIf to include patchFailedAPITests.
(CompileWebKitToT.doStepIf): Ditto.
(RunAPITests.evaluateCommand): Check if tests failed and retry them if required.
(ReRunAPITests): Re-run API tests.
(ReRunAPITests.evaluateCommand): Check if tests failed and retry on clean build if required.
(RunAPITestsWithoutPatch): Run API tests without patch.
(RunAPITestsWithoutPatch.doStepIf):
(RunAPITestsWithoutPatch.hideStepIf):
(RunAPITestsWithoutPatch.evaluateCommand):
(AnalyzeAPITestsResults): Analyze API test results from previous runs.
(AnalyzeAPITestsResults.start):
(AnalyzeAPITestsResults.analyzeResults): Analyze API test results.
(AnalyzeAPITestsResults.getBuildStepByName): Search for a build step by name.
(AnalyzeAPITestsResults.getTestsResults): Get the test results from previous API tesst steps.
* BuildSlaveSupport/ews-build/steps_unittest.py: Monkey patched FakeBuild.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@243342 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/ews-build/steps.py
Tools/BuildSlaveSupport/ews-build/steps_unittest.py
Tools/ChangeLog

index b985be4..e720d13 100644 (file)
@@ -28,6 +28,7 @@ from buildbot.steps.source import git
 from buildbot.steps.worker import CompositeStepMixin
 from twisted.internet import defer
 
+import json
 import re
 import requests
 
@@ -376,7 +377,7 @@ class UnApplyPatchIfRequired(CleanWorkingDirectory):
     descriptionDone = ['Unapplied patch']
 
     def doStepIf(self, step):
-        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests')
+        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests') or self.getProperty('patchFailedAPITests')
 
     def hideStepIf(self, results, step):
         return not self.doStepIf(step)
@@ -526,7 +527,7 @@ class CompileWebKitToT(CompileWebKit):
     haltOnFailure = True
 
     def doStepIf(self, step):
-        return self.getProperty('patchFailedToBuild')
+        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedAPITests')
 
     def hideStepIf(self, results, step):
         return not self.doStepIf(step)
@@ -710,6 +711,139 @@ class RunAPITests(TestWithFailureCount):
             return 0
         return int(match.group('ran')) - int(match.group('passed'))
 
+    def evaluateCommand(self, cmd):
+        rc = super(RunAPITests, self).evaluateCommand(cmd)
+        if rc == SUCCESS:
+            message = 'Passed API tests'
+            self.descriptionDone = message
+            self.build.results = SUCCESS
+            self.build.buildFinished([message], SUCCESS)
+        else:
+            self.build.addStepsAfterCurrentStep([ReRunAPITests()])
+        return rc
+
+
+class ReRunAPITests(RunAPITests):
+    name = 're-run-api-tests'
+
+    def evaluateCommand(self, cmd):
+        rc = TestWithFailureCount.evaluateCommand(self, cmd)
+        if rc == SUCCESS:
+            message = 'Passed API tests'
+            self.descriptionDone = message
+            self.build.results = SUCCESS
+            self.build.buildFinished([message], SUCCESS)
+        else:
+            self.setProperty('patchFailedAPITests', True)
+            self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileWebKitToT(), RunAPITestsWithoutPatch(), AnalyzeAPITestsResults()])
+        return rc
+
+
+class RunAPITestsWithoutPatch(RunAPITests):
+    name = 'run-api-tests-without-patch'
+
+    def evaluateCommand(self, cmd):
+        return TestWithFailureCount.evaluateCommand(self, cmd)
+
+
+class AnalyzeAPITestsResults(buildstep.BuildStep):
+    name = 'analyze-api-tests-results'
+    description = ['analyze-api-test-results']
+    descriptionDone = ['analyze-api-tests-results']
+
+    def start(self):
+        self.results = {}
+        d = self.getTestsResults(RunAPITests.name)
+        d.addCallback(lambda res: self.getTestsResults(ReRunAPITests.name))
+        d.addCallback(lambda res: self.getTestsResults(RunAPITestsWithoutPatch.name))
+        d.addCallback(lambda res: self.analyzeResults())
+        return defer.succeed(None)
+
+    def analyzeResults(self):
+        if not self.results or len(self.results) == 0:
+            self._addToLog('stderr', 'Unable to parse API test results: {}'.format(self.results))
+            self.finished(RETRY)
+            self.build.buildFinished(['Unable to parse API test results'], RETRY)
+            return -1
+
+        first_run_results = self.results.get(RunAPITests.name)
+        second_run_results = self.results.get(ReRunAPITests.name)
+        clean_tree_results = self.results.get(RunAPITestsWithoutPatch.name)
+
+        if not (first_run_results and second_run_results and clean_tree_results):
+            self.finished(RETRY)
+            self.build.buildFinished(['Unable to parse API test results'], RETRY)
+            return -1
+
+        def getAPITestFailures(result):
+            # TODO: Analyze Time-out, Crash and Failure independently
+            return set([failure.get('name') for failure in result.get('Timedout', [])] +
+                [failure.get('name') for failure in result.get('Crashed', [])] +
+                [failure.get('name') for failure in result.get('Failed', [])])
+
+        first_run_failures = getAPITestFailures(first_run_results)
+        second_run_failures = getAPITestFailures(second_run_results)
+        clean_tree_failures = getAPITestFailures(clean_tree_results)
+
+        self._addToLog('stderr', '\nFailures in API Test first run: {}'.format(first_run_failures))
+        self._addToLog('stderr', '\nFailures in API Test second run: {}'.format(first_run_failures))
+        self._addToLog('stderr', '\nFailures in API Test on clean tree: {}'.format(clean_tree_failures))
+        failures_with_patch = first_run_failures.intersection(second_run_failures)
+        new_failures = failures_with_patch - clean_tree_failures
+        new_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in new_failures])
+
+        if new_failures:
+            self._addToLog('stderr', '\nNew failures: {}\n'.format(new_failures))
+            self.finished(FAILURE)
+            self.build.results = FAILURE
+            message = 'Found {} new API Tests failures: {}'.format(len(new_failures), new_failures_string)
+            self.descriptionDone = message
+            self.build.buildFinished([message], FAILURE)
+        else:
+            self._addToLog('stderr', '\nNo new failures\n')
+            self.finished(SUCCESS)
+            self.build.results = SUCCESS
+            self.descriptionDone = 'Passed API tests'
+            message = 'Found {} pre-existing API tests failures'.format(len(clean_tree_failures))
+            self.build.buildFinished([message], SUCCESS)
+
+    @defer.inlineCallbacks
+    def _addToLog(self, logName, message):
+        try:
+            log = self.getLog(logName)
+        except KeyError:
+            log = yield self.addLog(logName)
+        log.addStdout(message)
+
+    def getBuildStepByName(self, name):
+        for step in self.build.executedSteps:
+            if step.name == name:
+                return step
+        return None
+
+    @defer.inlineCallbacks
+    def getTestsResults(self, name):
+        step = self.getBuildStepByName(name)
+        if not step:
+            self._addToLog('stderr', 'ERROR: step not found: {}'.format(step))
+            defer.returnValue(None)
+
+        logs = yield self.master.db.logs.getLogs(step.stepid)
+        log = next((log for log in logs if log['name'] == u'json'), None)
+        if not log:
+            self._addToLog('stderr', 'ERROR: log for step not found: {}'.format(step))
+            defer.returnValue(None)
+
+        lastline = int(max(0, log['num_lines'] - 1))
+        logLines = yield self.master.db.logs.getLogLines(log['id'], 0, lastline)
+        if log['type'] == 's':
+            logLines = ''.join([line[1:] for line in logLines.splitlines()])
+
+        try:
+            self.results[name] = json.loads(logLines)
+        except Exception as ex:
+            self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
+
 
 class ArchiveTestResults(shell.ShellCommand):
     command = ['python', 'Tools/BuildSlaveSupport/test-result-archive',
index b0fb079..76f19cd 100644 (file)
@@ -36,6 +36,10 @@ from twisted.trial import unittest
 
 from steps import *
 
+# Workaround for https://github.com/buildbot/buildbot/issues/4669
+from buildbot.test.fake.fakebuild import FakeBuild
+FakeBuild.addStepsAfterCurrentStep = lambda FakeBuild, step_factories: None
+
 
 class ExpectMasterShellCommand(object):
     def __init__(self, command, workdir=None, env=None, usePTY=0):
index 3f4aa8a..e834fc4 100644 (file)
@@ -1,5 +1,29 @@
 2019-03-21  Aakash Jain  <aakash_jain@apple.com>
 
+        [ews-build] Retry API test in case of failures
+        https://bugs.webkit.org/show_bug.cgi?id=196004
+
+        Reviewed by Lucas Forschler.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (UnApplyPatchIfRequired.doStepIf): Updated doStepIf to include patchFailedAPITests.
+        (CompileWebKitToT.doStepIf): Ditto.
+        (RunAPITests.evaluateCommand): Check if tests failed and retry them if required.
+        (ReRunAPITests): Re-run API tests.
+        (ReRunAPITests.evaluateCommand): Check if tests failed and retry on clean build if required.
+        (RunAPITestsWithoutPatch): Run API tests without patch.
+        (RunAPITestsWithoutPatch.doStepIf):
+        (RunAPITestsWithoutPatch.hideStepIf):
+        (RunAPITestsWithoutPatch.evaluateCommand):
+        (AnalyzeAPITestsResults): Analyze API test results from previous runs.
+        (AnalyzeAPITestsResults.start):
+        (AnalyzeAPITestsResults.analyzeResults): Analyze API test results.
+        (AnalyzeAPITestsResults.getBuildStepByName): Search for a build step by name.
+        (AnalyzeAPITestsResults.getTestsResults): Get the test results from previous API tesst steps.
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Monkey patched FakeBuild.
+
+2019-03-21  Aakash Jain  <aakash_jain@apple.com>
+
         [ews-build] Update queues configurations
         https://bugs.webkit.org/show_bug.cgi?id=196074