[ews-build] Add build step to run API Tests
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 26 Jul 2018 22:51:39 +0000 (22:51 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 26 Jul 2018 22:51:39 +0000 (22:51 +0000)
https://bugs.webkit.org/show_bug.cgi?id=188079

Reviewed by Lucas Forschler.

* BuildSlaveSupport/ews-build/steps.py:
(TestWithFailureCount): Common class for various tests with failure count.
(TestWithFailureCount.start): Configure Buildbot's logobserver.
(TestWithFailureCount.commandComplete): Updates the failure count.
(TestWithFailureCount.evaluateCommand): Set the step status appropriately.
(TestWithFailureCount.getResultSummary): Set the step summary appropriately.
(RunAPITests): Build step to run API tests.
(RunAPITests.countFailures): Count the failures by parsing the output.
* BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@234280 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/ews-build/steps.py
Tools/BuildSlaveSupport/ews-build/steps_unittest.py
Tools/ChangeLog

index bebaf22..5ced990 100644 (file)
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from buildbot.process import buildstep, properties
+from buildbot.process import buildstep, logobserver, properties
 from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
 from buildbot.steps import shell, transfer
 from buildbot.steps.source import svn
 from twisted.internet import defer
 
+import re
+
 EWS_URL = 'http://ews-build.webkit-uat.org/'
 WithProperties = properties.WithProperties
 
@@ -89,6 +91,44 @@ class CheckStyle(shell.ShellCommand):
     command = ['Tools/Scripts/check-webkit-style']
 
 
+class TestWithFailureCount(shell.Test):
+    failedTestsFormatString = "%d test%s failed"
+    failedTestCount = 0
+
+    def start(self):
+        self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
+        self.addLogObserver('stdio', self.log_observer)
+        return shell.Test.start(self)
+
+    def countFailures(self, cmd):
+        raise NotImplementedError
+
+    def commandComplete(self, cmd):
+        shell.Test.commandComplete(self, cmd)
+        self.failedTestCount = self.countFailures(cmd)
+        self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
+
+    def evaluateCommand(self, cmd):
+        if self.failedTestCount:
+            return FAILURE
+
+        if cmd.rc != 0:
+            return FAILURE
+
+        return SUCCESS
+
+    def getResultSummary(self):
+        status = self.name
+
+        if self.results != SUCCESS and self.failedTestCount:
+            status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)
+
+        if self.results != SUCCESS:
+            status += u' ({})'.format(Results[self.results])
+
+        return {u'step': status}
+
+
 class RunBindingsTests(shell.ShellCommand):
     name = 'bindings-tests'
     description = ['bindings-tests running']
@@ -330,3 +370,23 @@ class ExtractBuiltProduct(shell.ShellCommand):
     descriptionDone = ['extracted built product']
     haltOnFailure = True
     flunkOnFailure = True
+
+
+class RunAPITests(TestWithFailureCount):
+    name = 'run-api-tests'
+    description = ['api tests running']
+    descriptionDone = ['api-tests']
+    command = ['python', 'Tools/Scripts/run-api-tests', '--no-build', WithProperties('--%(configuration)s'), '--verbose']
+    failedTestsFormatString = '%d api test%s failed or timed out'
+
+    def start(self):
+        appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
+        return TestWithFailureCount.start(self)
+
+    def countFailures(self, cmd):
+        log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
+
+        match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
+        if not match:
+            return 0
+        return int(match.group('ran')) - int(match.group('passed'))
index 8edbd44..a21063b 100644 (file)
@@ -858,5 +858,203 @@ class TestExtractBuiltProduct(BuildStepMixinAdditions, unittest.TestCase):
         return self.runStep()
 
 
+class TestRunAPITests(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_success_mac(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'mac-mojave')
+        self.setProperty('platform', 'mac')
+        self.setProperty('configuration', 'release')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--release', '--verbose'],
+                        )
+            + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+Ran 1888 tests of 1888 with 1888 successful
+------------------------------
+All tests successfully passed!
+''')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+        return self.runStep()
+
+    def test_success_ios_simulator(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'ios-simulator-11')
+        self.setProperty('platform', 'ios')
+        self.setProperty('configuration', 'debug')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--ios-simulator'],
+                        )
+            + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+Ran 1888 tests of 1888 with 1888 successful
+------------------------------
+All tests successfully passed!
+''')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+        return self.runStep()
+
+    def test_one_failure(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'mac-mojave')
+        self.setProperty('platform', 'mac')
+        self.setProperty('configuration', 'debug')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+                        )
+            + ExpectShell.log('stdio', stdout='''
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1887 successful
+------------------------------
+Test suite failed
+
+Crashed
+
+    TestWTF.WTF.StringConcatenate_Unsigned
+        **FAIL** WTF.StringConcatenate_Unsigned
+
+        Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+        Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
+          Actual: hello 42 world
+        Expected: 'hello * world'
+        Which is: 74B00C9C
+
+Testing completed, Exit status: 3
+''')
+            + 1,
+        )
+        self.expectOutcome(result=FAILURE, state_string='1 api test failed or timed out (failure)')
+        return self.runStep()
+
+    def test_multiple_failures_and_timeouts(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'mac-mojave')
+        self.setProperty('platform', 'mac')
+        self.setProperty('configuration', 'debug')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+                        )
+            + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1884 successful
+------------------------------
+Test suite failed
+
+Failed
+
+    TestWTF.WTF.StringConcatenate_Unsigned
+        **FAIL** WTF.StringConcatenate_Unsigned
+
+        Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+        Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
+          Actual: hello 42 world
+        Expected: 'hello * world'
+        Which is: 74B00C9C
+
+    TestWTF.WTF_Expected.Unexpected
+        **FAIL** WTF_Expected.Unexpected
+
+        Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
+        Value of: s1
+          Actual: oops
+        Expected: s0
+        Which is: oops
+
+Timeout
+
+    TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
+    TestWTF.WTF_Lock.ContendedShortSection
+
+Testing completed, Exit status: 3
+''')
+            + 4,
+        )
+        self.expectOutcome(result=FAILURE, state_string='4 api tests failed or timed out (failure)')
+        return self.runStep()
+
+    def test_unexpecte_failure(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'mac-mojave')
+        self.setProperty('platform', 'mac')
+        self.setProperty('configuration', 'debug')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+                        )
+            + ExpectShell.log('stdio', stdout='Unexpected failure. Failed to run api tests.')
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='run-api-tests (failure)')
+        return self.runStep()
+
+    def test_no_failures_or_timeouts_with_disabled(self):
+        self.setupStep(RunAPITests())
+        self.setProperty('fullPlatform', 'mac-mojave')
+        self.setProperty('platform', 'mac')
+        self.setProperty('configuration', 'debug')
+
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+                        )
+            + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1881 tests of 1888 with 1881 successful
+------------------------------
+All tests successfully passed!
+''')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+        return self.runStep()
+
+
 if __name__ == '__main__':
     unittest.main()
index b787df3..28e437a 100644 (file)
@@ -1,3 +1,20 @@
+2018-07-26  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews-build] Add build step to run API Tests
+        https://bugs.webkit.org/show_bug.cgi?id=188079
+
+        Reviewed by Lucas Forschler.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (TestWithFailureCount): Common class for various tests with failure count.
+        (TestWithFailureCount.start): Configure Buildbot's logobserver.
+        (TestWithFailureCount.commandComplete): Updates the failure count.
+        (TestWithFailureCount.evaluateCommand): Set the step status appropriately.
+        (TestWithFailureCount.getResultSummary): Set the step summary appropriately.
+        (RunAPITests): Build step to run API tests.
+        (RunAPITests.countFailures): Count the failures by parsing the output.
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-tests.
+
 2018-07-26  Ross Kirsling  <ross.kirsling@sony.com>
 
         String(View) should have a splitAllowingEmptyEntries function instead of a flag parameter