Add support for test262 JavaScriptCore tests

https://bugs.webkit.org/show_bug.cgi?id=170523

Reviewed by Daniel Bates.

* BuildSlaveSupport/build.webkit.org-config/config.json: Added bot433 and bot434.
* BuildSlaveSupport/build.webkit.org-config/master.cfg:
(RunTest262Tests): Added class to run Test262 tests.
(RunTest262Tests.start): Added.
(RunTest262Tests.countFailures): Method to count the failures.
(Test262Factory): Added Test262 factory class.
* BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py:
(RunTest262TestsTest): Added unit tests.
(RunTest262TestsTest.assertResults): Helper method.
(RunTest262TestsTest.test_no_regressions_output): Added test case.
(test_failure_output): Same.
(test_failures_output): Same.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215108 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/config.json b/Tools/BuildSlaveSupport/build.webkit.org-config/config.json
index 5987ede..9e7fe2f 100644 (file)
--- a/Tools/BuildSlaveSupport/build.webkit.org-config/config.json
+++ b/Tools/BuildSlaveSupport/build.webkit.org-config/config.json
@@ -20,6 +20,8 @@
                     { "name": "bot417", "platform": "mac-elcapitan" },
                     { "name": "bot418", "platform": "mac-elcapitan" },
                     { "name": "bot419", "platform": "mac-elcapitan" },
                     { "name": "bot417", "platform": "mac-elcapitan" },
                     { "name": "bot418", "platform": "mac-elcapitan" },
                     { "name": "bot419", "platform": "mac-elcapitan" },

+                    { "name": "bot433", "platform": "mac-elcapitan" },
+                    { "name": "bot434", "platform": "mac-elcapitan" },

 
                     { "name": "bot118", "platform": "mac-sierra" },
                     { "name": "bot121", "platform": "mac-sierra" },
 
                     { "name": "bot118", "platform": "mac-sierra" },
                     { "name": "bot121", "platform": "mac-sierra" },


@@ -138,7 +140,7 @@
                     },
                     { "name": "Apple El Capitan Debug (Build)", "type": "Build", "builddir": "elcapitan-debug",
                       "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"],
                     },
                     { "name": "Apple El Capitan Debug (Build)", "type": "Build", "builddir": "elcapitan-debug",
                       "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"],

-                      "triggers": ["elcapitan-debug-tests-jsc", "elcapitan-debug-tests-wk1", "elcapitan-debug-tests-wk2"],
+                      "triggers": ["elcapitan-debug-tests-jsc", "elcapitan-debug-tests-test262", "elcapitan-debug-tests-wk1", "elcapitan-debug-tests-wk2"],

                       "slavenames": ["bot415", "bot416"]
                     },
                     { "name": "Apple El Capitan CMake Debug (Build)", "type": "Build", "builddir": "elcapitan-cmake-debug",
                       "slavenames": ["bot415", "bot416"]
                     },
                     { "name": "Apple El Capitan CMake Debug (Build)", "type": "Build", "builddir": "elcapitan-cmake-debug",


@@ -160,6 +162,10 @@
                       "additionalArguments": ["--no-retry-failures", "--no-sample-on-timeout"],
                       "slavenames": ["bot162", "bot165"]
                     },
                       "additionalArguments": ["--no-retry-failures", "--no-sample-on-timeout"],
                       "slavenames": ["bot162", "bot165"]
                     },

+                    { "name": "Apple El Capitan Debug Test262 (Tests)", "type": "Test262", "builddir": "elcapitan-debug-tests-test262",
+                      "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"],
+                      "slavenames": ["bot433"]
+                    },

                     { "name": "Apple El Capitan LLINT CLoop (BuildAndTest)", "type": "BuildAndTestLLINTCLoop", "builddir": "elcapitan-cloop-debug",
                       "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"],
                       "slavenames": ["bot169"]
                     { "name": "Apple El Capitan LLINT CLoop (BuildAndTest)", "type": "BuildAndTestLLINTCLoop", "builddir": "elcapitan-cloop-debug",
                       "platform": "mac-elcapitan", "configuration": "debug", "architectures": ["x86_64"],
                       "slavenames": ["bot169"]


@@ -170,7 +176,7 @@
                     },
                     { "name": "Apple El Capitan Release (Build)", "type": "Build", "builddir": "elcapitan-release",
                       "platform": "mac-elcapitan", "configuration": "release", "architectures": ["x86_64"],
                     },
                     { "name": "Apple El Capitan Release (Build)", "type": "Build", "builddir": "elcapitan-release",
                       "platform": "mac-elcapitan", "configuration": "release", "architectures": ["x86_64"],

-                      "triggers": ["elcapitan-release-tests-jsc", "elcapitan-release-tests-wk1", "elcapitan-release-tests-wk2", "elcapitan-release-perf"],
+                      "triggers": ["elcapitan-release-tests-jsc", "elcapitan-release-tests-test262", "elcapitan-release-tests-wk1", "elcapitan-release-tests-wk2", "elcapitan-release-perf"],

                       "slavenames": ["bot417", "bot418"]
                     },
                     { "name": "Apple El Capitan Release JSC (Tests)", "type": "TestJSC", "builddir": "elcapitan-release-tests-jsc",
                       "slavenames": ["bot417", "bot418"]
                     },
                     { "name": "Apple El Capitan Release JSC (Tests)", "type": "TestJSC", "builddir": "elcapitan-release-tests-jsc",


@@ -196,6 +202,10 @@
                       "platform": "mac-elcapitan", "configuration": "release", "architectures": ["i386"],
                       "slavenames": ["bot419"]
                     },
                       "platform": "mac-elcapitan", "configuration": "release", "architectures": ["i386"],
                       "slavenames": ["bot419"]
                     },

+                    { "name": "Apple El Capitan Release Test262 (Tests)", "type": "Test262", "builddir": "elcapitan-release-tests-test262",
+                      "platform": "mac-elcapitan", "configuration": "release", "architectures": ["x86_64"],
+                      "slavenames": ["bot434"]
+                    },

                     {
                       "name": "Apple iOS 10 Release (Build)", "type": "Build", "builddir": "ios-10-release",
                       "platform": "ios-10", "configuration": "release", "architectures": ["armv7s", "arm64"],
                     {
                       "name": "Apple iOS 10 Release (Build)", "type": "Build", "builddir": "ios-10-release",
                       "platform": "ios-10", "configuration": "release", "architectures": ["armv7s", "arm64"],


@@ -403,6 +413,12 @@
                     { "type": "Triggerable", "name": "elcapitan-release-perf",
                       "builderNames": ["Apple El Capitan Release WK2 (Perf)"]
                     },
                     { "type": "Triggerable", "name": "elcapitan-release-perf",
                       "builderNames": ["Apple El Capitan Release WK2 (Perf)"]
                     },

+                    { "type": "Triggerable", "name": "elcapitan-debug-tests-test262",
+                      "builderNames": ["Apple El Capitan Debug Test262 (Tests)"]
+                    },
+                    { "type": "Triggerable", "name": "elcapitan-release-tests-test262",
+                      "builderNames": ["Apple El Capitan Release Test262 (Tests)"]
+                    },

                     { "type": "Triggerable", "name": "ios-simulator-10-release-tests-wk1",
                       "builderNames": ["Apple iOS 10 Simulator Release WK1 (Tests)"]
                     },
                     { "type": "Triggerable", "name": "ios-simulator-10-release-tests-wk1",
                       "builderNames": ["Apple iOS 10 Simulator Release WK1 (Tests)"]
                     },

diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg
index 88e037a..7e1e85a 100644 (file)
--- a/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg
+++ b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg
@@ -322,6 +322,25 @@ class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
         return RunJavaScriptCoreTests.start(self)
 
 
         return RunJavaScriptCoreTests.start(self)
 
 

+class RunTest262Tests(TestWithFailureCount):
+    name = "test262-test"
+    description = ["test262-tests running"]
+    descriptionDone = ["test262-tests"]
+    failedTestsFormatString = "%d Test262 test%s failed"
+    command = ["perl", "./Tools/Scripts/run-jsc-stress-tests", WithProperties("--%(configuration)s"), "JSTests/test262.yaml"]
+
+    def start(self):
+        appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
+        return shell.Test.start(self)
+
+    def countFailures(self, cmd):
+        logText = cmd.logs['stdio'].getText()
+        matches = re.findall(r'\(failed (\d+)\)', logText)
+        if matches:
+            return int(matches[-1])
+        return 0
+
+

 class RunWebKitTests(shell.Test):
     name = "layout-test"
     description = ["layout-tests running"]
 class RunWebKitTests(shell.Test):
     name = "layout-test"
     description = ["layout-tests running"]


@@ -893,6 +912,13 @@ class TestJSCFactory(Factory):
         self.addStep(ExtractBuiltProduct())
         self.addStep(RunJavaScriptCoreTests())
 
         self.addStep(ExtractBuiltProduct())
         self.addStep(RunJavaScriptCoreTests())
 

+class Test262Factory(Factory):
+    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
+        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
+        self.addStep(DownloadBuiltProduct())
+        self.addStep(ExtractBuiltProduct())
+        self.addStep(RunTest262Tests())
+

 class TestWebKit1Factory(TestFactory):
     LayoutTestClass = RunWebKit1Tests
 
 class TestWebKit1Factory(TestFactory):
     LayoutTestClass = RunWebKit1Tests
 

diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py b/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
index 1135908..66e36c3 100755 (executable)
--- a/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
+++ b/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
@@ -140,6 +140,34 @@ class RunJavaScriptCoreTestsTest(unittest.TestCase):
     5 failures found.""")
 
 
     5 failures found.""")
 
 

+class RunTest262TestsTest(unittest.TestCase):
+    def assertResults(self, expected_result, expected_text, rc, stdio):
+        cmd = StubRemoteCommand(rc, stdio)
+        step = RunTest262Tests()
+        step.commandComplete(cmd)
+        actual_results = step.evaluateCommand(cmd)
+        actual_text = step.getText2(cmd, actual_results)
+
+        self.assertEqual(expected_result, actual_results)
+        self.assertEqual(actual_text, expected_text)
+
+    def test_no_regressions_output(self):
+        self.assertResults(SUCCESS, ["test262-test"], 0, """Using the following jsc path: /WebKitBuild/Release/jsc
+168/168         """)
+
+    def test_failure_output(self):
+        self.assertResults(FAILURE, ["1 Test262 test failed"], 1, """Using the following jsc path: /WebKitBuild/Release/jsc
+test262.yaml/test262/test/built-ins/Array/from/iter-set-elem-prop-err.js.default: ERROR: Unexpected exit code: 0
+test262.yaml/test262/test/built-ins/Array/from/iter-set-elem-prop-err.js.default-strict: ERROR: Unexpected exit code: 0
+...
+43768/43768 (failed 1)         """)
+
+    def test_failures_output(self):
+        self.assertResults(FAILURE, ["75 Test262 tests failed"], 75, """Using the following jsc path: /WebKitBuild/Release/jsc
+...
+43768/43768 (failed 75)         """)
+
+

 class RunLLINTCLoopTestsTest(unittest.TestCase):
     def assertResults(self, expected_result, expected_text, rc, stdio):
         cmd = StubRemoteCommand(rc, stdio)
 class RunLLINTCLoopTestsTest(unittest.TestCase):
     def assertResults(self, expected_result, expected_text, rc, stdio):
         cmd = StubRemoteCommand(rc, stdio)


@@ -374,6 +402,7 @@ expected_build_steps = {
     'Apple El Capitan CMake Debug (Build)' :['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
     'Apple El Capitan Debug (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'],
     'Apple El Capitan Debug JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
     'Apple El Capitan CMake Debug (Build)' :['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
     'Apple El Capitan Debug (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'],
     'Apple El Capitan Debug JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],

+    'Apple El Capitan Debug Test262 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],

     'Apple El Capitan Debug WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan Debug WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan LLINT CLoop (BuildAndTest)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
     'Apple El Capitan Debug WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan Debug WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan LLINT CLoop (BuildAndTest)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],


@@ -381,6 +410,7 @@ expected_build_steps = {
     'Apple El Capitan Release (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'],
     'Apple El Capitan Release WK2 (Perf)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'],
     'Apple El Capitan Release JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
     'Apple El Capitan Release (Build)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'trigger'],
     'Apple El Capitan Release WK2 (Perf)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'],
     'Apple El Capitan Release JSC (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],

+    'Apple El Capitan Release Test262 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],

     'Apple El Capitan Release WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan Release WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
 
     'Apple El Capitan Release WK1 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
     'Apple El Capitan Release WK2 (Tests)' : ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
 

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index dfc5c7e..72e0db9 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,23 @@
+2017-04-07  Aakash Jain  <aakash_jain@apple.com>
+
+        Add support for test262 JavaScriptCore tests
+        https://bugs.webkit.org/show_bug.cgi?id=170523
+
+        Reviewed by Daniel Bates.
+
+        * BuildSlaveSupport/build.webkit.org-config/config.json: Added bot433 and bot434.
+        * BuildSlaveSupport/build.webkit.org-config/master.cfg:
+        (RunTest262Tests): Added class to run Test262 tests.
+        (RunTest262Tests.start): Added.
+        (RunTest262Tests.countFailures): Method to count the failures.
+        (Test262Factory): Added Test262 factory class.
+        * BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py:
+        (RunTest262TestsTest): Added unit tests.
+        (RunTest262TestsTest.assertResults): Helper method.
+        (RunTest262TestsTest.test_no_regressions_output): Added test case.
+        (test_failure_output): Same. 
+        (test_failures_output): Same.
+

 2017-04-07  Caio Lima  <ticaiolima@gmail.com>
 
         [JSC] Change --debug option to --debugger
 2017-04-07  Caio Lima  <ticaiolima@gmail.com>
 
         [JSC] Change --debug option to --debugger