From d761cd98ed36d32dd564b0ec094f6bf9492599d8 Mon Sep 17 00:00:00 2001 From: "aakash_jain@apple.com" Date: Fri, 7 Apr 2017 19:06:51 +0000 Subject: [PATCH] Add support for test262 JavaScriptCore tests https://bugs.webkit.org/show_bug.cgi?id=170523 Reviewed by Daniel Bates. * BuildSlaveSupport/build.webkit.org-config/config.json: Added bot433 and bot434. * BuildSlaveSupport/build.webkit.org-config/master.cfg: (RunTest262Tests): Added class to run Test262 tests. (RunTest262Tests.start): Added. (RunTest262Tests.countFailures): Method to count the failures. (Test262Factory): Added Test262 factory class. * BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py: (RunTest262TestsTest): Added unit tests. (RunTest262TestsTest.assertResults): Helper method. (RunTest262TestsTest.test_no_regressions_output): Added test case. (test_failure_output): Same. (test_failures_output): Same. git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215108 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- .../build.webkit.org-config/config.json | 20 +++++++++++++-- .../build.webkit.org-config/master.cfg | 26 +++++++++++++++++++ .../build.webkit.org-config/mastercfg_unittest.py | 30 ++++++++++++++++++++++ Tools/ChangeLog | 20 +++++++++++++++ 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/config.json b/Tools/BuildSlaveSupport/build.webkit.org-config/config.json index 5987ede..9e7fe2f 100644 --- a/Tools/BuildSlaveSupport/build.webkit.org-config/config.json +++ b/Tools/BuildSlaveSupport/build.webkit.org-config/config.json @@ -20,6 +20,8 @@ { "name": "bot417", "platform": "mac-elcapitan" }, { "name": "bot418", "platform": "mac-elcapitan" }, { "name": "bot419", "platform": "mac-elcapitan" }, + { "name": "bot433", "platform": "mac-elcapitan" }, + { "name": "bot434", "platform": "mac-elcapitan" }, { "name": "bot118", "platform": "mac-sierra" }, { "name": "bot121", "platform": "mac-sierra" }, @@ -138,7 +140,7 @@ }, { "name": "Apple El Capitan Debug (Build)", "type": "Build", "builddir": "elcapitan-debug", "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"], - "triggers": ["elcapitan-debug-tests-jsc", "elcapitan-debug-tests-wk1", "elcapitan-debug-tests-wk2"], + "triggers": ["elcapitan-debug-tests-jsc", "elcapitan-debug-tests-test262", "elcapitan-debug-tests-wk1", "elcapitan-debug-tests-wk2"], "slavenames": ["bot415", "bot416"] }, { "name": "Apple El Capitan CMake Debug (Build)", "type": "Build", "builddir": "elcapitan-cmake-debug", @@ -160,6 +162,10 @@ "additionalArguments": ["--no-retry-failures", "--no-sample-on-timeout"], "slavenames": ["bot162", "bot165"] }, + { "name": "Apple El Capitan Debug Test262 (Tests)", "type": "Test262", "builddir": "elcapitan-debug-tests-test262", + "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"], + "slavenames": ["bot433"] + }, { "name": "Apple El Capitan LLINT CLoop (BuildAndTest)", "type": "BuildAndTestLLINTCLoop", "builddir": "elcapitan-cloop-debug", "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"], "slavenames": ["bot169"] @@ -170,7 +176,7 @@ }, { "name": "Apple El Capitan Release (Build)", "type": "Build", "builddir": "elcapitan-release", "platform": "mac-elcapitan", "configuration": "release", "architectures": ["x86_64"], - "triggers": ["elcapitan-release-tests-jsc", "elcapitan-release-tests-wk1", "elcapitan-release-tests-wk2", "elcapitan-release-perf"], + "triggers": ["elcapitan-release-tests-jsc", "elcapitan-release-tests-test262", "elcapitan-release-tests-wk1", "elcapitan-release-tests-wk2", "elcapitan-release-perf"], "slavenames": ["bot417", "bot418"] }, { "name": "Apple El Capitan Release JSC (Tests)", "type": "TestJSC", "builddir": "elcapitan-release-tests-jsc", @@ -196,6 +202,10 @@ "platform": "mac-elcapitan", "configuration": "release", "architectures": ["i386"], "slavenames": ["bot419"] }, + { "name": "Apple El Capitan Release Test262 (Tests)", "type": "Test262", "builddir": "elcapitan-release-tests-test262", + "platform": "mac-elcapitan", "configuration": "release", "architectures": ["x86_64"], + "slavenames": ["bot434"] + }, { "name": "Apple iOS 10 Release (Build)", "type": "Build", "builddir": "ios-10-release", "platform": "ios-10", "configuration": "release", "architectures": ["armv7s", "arm64"], @@ -403,6 +413,12 @@ { "type": "Triggerable", "name": "elcapitan-release-perf", "builderNames": ["Apple El Capitan Release WK2 (Perf)"] }, + { "type": "Triggerable", "name": "elcapitan-debug-tests-test262", + "builderNames": ["Apple El Capitan Debug Test262 (Tests)"] + }, + { "type": "Triggerable", "name": "elcapitan-release-tests-test262", + "builderNames": ["Apple El Capitan Release Test262 (Tests)"] + }, { "type": "Triggerable", "name": "ios-simulator-10-release-tests-wk1", "builderNames": ["Apple iOS 10 Simulator Release WK1 (Tests)"] }, diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg index 88e037a..7e1e85af 100644 --- a/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg +++ b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg @@ -322,6 +322,25 @@ class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests): return RunJavaScriptCoreTests.start(self) +class RunTest262Tests(TestWithFailureCount): + name = "test262-test" + description = ["test262-tests running"] + descriptionDone = ["test262-tests"] + failedTestsFormatString = "%d Test262 test%s failed" + command = ["perl", "./Tools/Scripts/run-jsc-stress-tests", WithProperties("--%(configuration)s"), "JSTests/test262.yaml"] + + def start(self): + appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform')) + return shell.Test.start(self) + + def countFailures(self, cmd): + logText = cmd.logs['stdio'].getText() + matches = re.findall(r'\(failed (\d+)\)', logText) + if matches: + return int(matches[-1]) + return 0 + + class RunWebKitTests(shell.Test): name = "layout-test" description = ["layout-tests running"] @@ -893,6 +912,13 @@ class TestJSCFactory(Factory): self.addStep(ExtractBuiltProduct()) self.addStep(RunJavaScriptCoreTests()) +class Test262Factory(Factory): + def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None): + Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror) + self.addStep(DownloadBuiltProduct()) + self.addStep(ExtractBuiltProduct()) + self.addStep(RunTest262Tests()) + class TestWebKit1Factory(TestFactory): LayoutTestClass = RunWebKit1Tests diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py b/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py index 1135908..66e36c3 100755 --- a/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py +++ b/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py @@ -140,6 +140,34 @@ class RunJavaScriptCoreTestsTest(unittest.TestCase): 5 failures found.""") +class RunTest262TestsTest(unittest.TestCase): + def assertResults(self, expected_result, expected_text, rc, stdio): + cmd = StubRemoteCommand(rc, stdio) + step = RunTest262Tests() + step.commandComplete(cmd) + actual_results = step.evaluateCommand(cmd) + actual_text = step.getText2(cmd, actual_results) + + self.assertEqual(expected_result, actual_results) + self.assertEqual(actual_text, expected_text) + + def test_no_regressions_output(self): + self.assertResults(SUCCESS, ["test262-test"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc +168/168 """) + + def test_failure_output(self): + self.assertResults(FAILURE, ["1 Test262 test failed"], 1, """Using the following jsc path: /WebKitBuild/Release/jsc +test262.yaml/test262/test/built-ins/Array/from/iter-set-elem-prop-err.js.default: ERROR: Unexpected exit code: 0 +test262.yaml/test262/test/built-ins/Array/from/iter-set-elem-prop-err.js.default-strict: ERROR: Unexpected exit code: 0 +... +43768/43768 (failed 1) """) + + def test_failures_output(self): + self.assertResults(FAILURE, ["75 Test262 tests failed"], 75, """Using the following jsc path: /WebKitBuild/Release/jsc +... +43768/43768 (failed 75) """) + + class RunLLINTCLoopTestsTest(unittest.TestCase): def assertResults(self, expected_result, expected_text, rc, stdio): cmd = StubRemoteCommand(rc, stdio) @@ -374,6 +402,7 @@ expected_build_steps = { 'Apple El Capitan CMake Debug (Build)' :['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'], 'Apple El Capitan Debug (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'], 'Apple El Capitan Debug JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'], + 'Apple El Capitan Debug Test262 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'], 'Apple El Capitan Debug WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'], 'Apple El Capitan Debug WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'], 'Apple El Capitan LLINT CLoop (BuildAndTest)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'], @@ -381,6 +410,7 @@ expected_build_steps = { 'Apple El Capitan Release (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'], 'Apple El Capitan Release WK2 (Perf)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'], 'Apple El Capitan Release JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'], + 'Apple El Capitan Release Test262 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'], 'Apple El Capitan Release WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'], 'Apple El Capitan Release WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'], diff --git a/Tools/ChangeLog b/Tools/ChangeLog index dfc5c7e..72e0db9 100644 --- a/Tools/ChangeLog +++ b/Tools/ChangeLog @@ -1,3 +1,23 @@ +2017-04-07 Aakash Jain + + Add support for test262 JavaScriptCore tests + https://bugs.webkit.org/show_bug.cgi?id=170523 + + Reviewed by Daniel Bates. + + * BuildSlaveSupport/build.webkit.org-config/config.json: Added bot433 and bot434. + * BuildSlaveSupport/build.webkit.org-config/master.cfg: + (RunTest262Tests): Added class to run Test262 tests. + (RunTest262Tests.start): Added. + (RunTest262Tests.countFailures): Method to count the failures. + (Test262Factory): Added Test262 factory class. + * BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py: + (RunTest262TestsTest): Added unit tests. + (RunTest262TestsTest.assertResults): Helper method. + (RunTest262TestsTest.test_no_regressions_output): Added test case. + (test_failure_output): Same. + (test_failures_output): Same. + 2017-04-07 Caio Lima [JSC] Change --debug option to --debugger -- 1.8.3.1