Add support for webkitpy tests EWS
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 7 Nov 2017 22:30:57 +0000 (22:30 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 7 Nov 2017 22:30:57 +0000 (22:30 +0000)
https://bugs.webkit.org/show_bug.cgi?id=178480

Reviewed by Alexey Proskuryakov.

* Scripts/webkitpy/common/config/ews.json: Added webkitpy EWS.
* Scripts/webkitpy/common/net/generictestresults.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults.py.
* Scripts/webkitpy/common/net/generictestresults_unittest.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults_unittest.py.
* Scripts/webkitpy/tool/bot/patchanalysistask.py:
(PatchAnalysisTask._test_patch): Do not retry webkitpy test since they are not flaky.
* Scripts/webkitpy/tool/bot/bindingstestresultsreader.py: Updated import for BindingsTestResult.
* Scripts/webkitpy/tool/bot/retrylogic_unittest.py: Ditto.
* Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py: Copied from Scripts/webkitpy/tool/bot/bindingstestresultsreader.py.
* Scripts/webkitpy/tool/commands/earlywarningsystem.py:
(AbstractEarlyWarningSystem.begin_work_queue): Added webkitpy test result reader.
* Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py: Updated unit-tests.
* Scripts/webkitpy/tool/steps/checkpatchrelevance.py: Run webkitpy EWS only for relevant changes.
* Scripts/webkitpy/tool/steps/runtests.py:
(RunTests.run): Run webkitpy tests when --group=webkitpy is passed.
(RunTests._run_webkitpy_tests): Method to invoke webkitpy tests.
* Scripts/webkitpy/tool/steps/steps_unittest.py:
(test_runtests_webkitpy): Added unit-test.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@224547 268f45cc-cd09-0410-ab3c-d52691b4dbfc

13 files changed:
Tools/ChangeLog
Tools/Scripts/webkitpy/common/config/ews.json
Tools/Scripts/webkitpy/common/net/generictestresults.py [moved from Tools/Scripts/webkitpy/common/net/bindingstestresults.py with 93% similarity]
Tools/Scripts/webkitpy/common/net/generictestresults_unittest.py [moved from Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py with 75% similarity]
Tools/Scripts/webkitpy/tool/bot/bindingstestresultsreader.py
Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py
Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py
Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py [new file with mode: 0644]
Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py
Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py
Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py
Tools/Scripts/webkitpy/tool/steps/runtests.py
Tools/Scripts/webkitpy/tool/steps/steps_unittest.py

index ed45ae0..3eaa0df 100644 (file)
@@ -1,3 +1,28 @@
+2017-11-07  Aakash Jain  <aakash_jain@apple.com>
+
+        Add support for webkitpy tests EWS
+        https://bugs.webkit.org/show_bug.cgi?id=178480
+
+        Reviewed by Alexey Proskuryakov.
+
+        * Scripts/webkitpy/common/config/ews.json: Added webkitpy EWS.
+        * Scripts/webkitpy/common/net/generictestresults.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults.py.
+        * Scripts/webkitpy/common/net/generictestresults_unittest.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults_unittest.py.
+        * Scripts/webkitpy/tool/bot/patchanalysistask.py:
+        (PatchAnalysisTask._test_patch): Do not retry webkitpy test since they are not flaky.
+        * Scripts/webkitpy/tool/bot/bindingstestresultsreader.py: Updated import for BindingsTestResult.
+        * Scripts/webkitpy/tool/bot/retrylogic_unittest.py: Ditto.
+        * Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py: Copied from Scripts/webkitpy/tool/bot/bindingstestresultsreader.py.
+        * Scripts/webkitpy/tool/commands/earlywarningsystem.py:
+        (AbstractEarlyWarningSystem.begin_work_queue): Added webkitpy test result reader.
+        * Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py: Updated unit-tests.
+        * Scripts/webkitpy/tool/steps/checkpatchrelevance.py: Run webkitpy EWS only for relevant changes.
+        * Scripts/webkitpy/tool/steps/runtests.py:
+        (RunTests.run): Run webkitpy tests when --group=webkitpy is passed.
+        (RunTests._run_webkitpy_tests): Method to invoke webkitpy tests.
+        * Scripts/webkitpy/tool/steps/steps_unittest.py:
+        (test_runtests_webkitpy): Added unit-test.
+
 2017-11-07  Commit Queue  <commit-queue@webkit.org>
 
         Unreviewed, rolling out r224512 and r224521.
index cea9947..342a6f8 100644 (file)
         "runTests": true,
         "shouldBuild": false
     },
+    "webkitpy EWS": {
+        "port": "mac",
+        "name": "webkitpy-ews",
+        "group": "webkitpy",
+        "runTests": true,
+        "shouldBuild": false
+    },
     "WPE EWS": {
         "port": "wpe",
         "watchers": [
@@ -27,7 +27,7 @@ from webkitpy.common.net.abstracttestresults import AbstractTestResults
 _log = logging.getLogger(__name__)
 
 
-class BindingsTestResults(AbstractTestResults):
+class GenericTestResults(AbstractTestResults):
     def __init__(self, failures):
         self._failures = failures
 
@@ -57,3 +57,11 @@ class BindingsTestResults(AbstractTestResults):
     # No defined failure limit for bindings tests.
     def did_exceed_test_failure_limit(self):
         return False
+
+
+class BindingsTestResults(GenericTestResults):
+    pass
+
+
+class WebkitpyTestResults(GenericTestResults):
+    pass
 
 import unittest
 
-from webkitpy.common.net.bindingstestresults import BindingsTestResults
+from webkitpy.common.net.generictestresults import GenericTestResults
 
 
-class BindingsTestResultsTest(unittest.TestCase):
+class GenericTestResultsTest(unittest.TestCase):
     def test_results_from_string(self):
         incomplete_json = '{"key2": []}'
-        self.assertEqual(None, BindingsTestResults.results_from_string(incomplete_json))
+        self.assertEqual(None, GenericTestResults.results_from_string(incomplete_json))
 
     def test_results_from_string_success(self):
         no_failures_string = '{"failures": []}'
-        no_failures_results = BindingsTestResults([])
-        self.assertTrue(no_failures_results.equals(BindingsTestResults.results_from_string(no_failures_string)))
+        no_failures_results = GenericTestResults([])
+        self.assertTrue(no_failures_results.equals(GenericTestResults.results_from_string(no_failures_string)))
         self.assertTrue(no_failures_results.all_passed())
 
         test_string = '{"failures": ["failure1"]}'
-        test_results = BindingsTestResults(["failure1"])
-        self.assertTrue(test_results.equals(BindingsTestResults.results_from_string(test_string)))
+        test_results = GenericTestResults(["failure1"])
+        self.assertTrue(test_results.equals(GenericTestResults.results_from_string(test_string)))
index 8fca2fd..525528b 100644 (file)
@@ -22,7 +22,7 @@
 
 import logging
 
-from webkitpy.common.net.bindingstestresults import BindingsTestResults
+from webkitpy.common.net.generictestresults import BindingsTestResults
 from webkitpy.tool.bot.abstracttestresultsreader import AbstractTestResultsReader
 
 _log = logging.getLogger(__name__)
index 5de8386..3ab2429 100644 (file)
@@ -368,6 +368,8 @@ class PatchAnalysisTask(object):
             return self._retry_jsc_tests()
         elif hasattr(self._delegate, 'group') and self._delegate.group() == "bindings":
             return self._retry_bindings_tests()
+        elif hasattr(self._delegate, 'group') and self._delegate.group() == "webkitpy":
+            return False
         else:
             return self._retry_layout_tests()
 
index 4c1b440..9a72c7c 100644 (file)
@@ -23,7 +23,7 @@
 import logging
 import unittest
 
-from webkitpy.common.net.bindingstestresults import BindingsTestResults
+from webkitpy.common.net.generictestresults import BindingsTestResults
 from webkitpy.common.net.jsctestresults import JSCTestResults
 from webkitpy.common.system.executive import ScriptError
 from webkitpy.tool.bot.patchanalysistask import *
diff --git a/Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py b/Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py
new file mode 100644 (file)
index 0000000..852ba85
--- /dev/null
@@ -0,0 +1,35 @@
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+
+from webkitpy.common.net.generictestresults import WebkitpyTestResults
+from webkitpy.tool.bot.abstracttestresultsreader import AbstractTestResultsReader
+
+_log = logging.getLogger(__name__)
+
+
+class WebkitpyTestResultsReader(AbstractTestResultsReader):
+    def results(self):
+        results_path = self._host.filesystem.join(self._results_directory, 'webkitpy_test_results.json')
+        contents = self._read_file_contents(results_path)
+        return WebkitpyTestResults.results_from_string(contents)
index e82b0a7..54a3a98 100644 (file)
@@ -39,6 +39,7 @@ from webkitpy.common.system.executive import ScriptError
 from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
 from webkitpy.tool.bot.bindingstestresultsreader import BindingsTestResultsReader
 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
+from webkitpy.tool.bot.webkitpytestresultsreader import WebkitpyTestResultsReader
 from webkitpy.tool.bot.jsctestresultsreader import JSCTestResultsReader
 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch, PatchIsNotValid, PatchIsNotApplicable
 from webkitpy.tool.bot.queueengine import QueueEngine
@@ -62,6 +63,8 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDele
             self._test_results_reader = JSCTestResultsReader(self._tool, self._port.jsc_results_directory())
         elif self.group() == "bindings":
             self._test_results_reader = BindingsTestResultsReader(self._tool, self._port.jsc_results_directory())
+        elif self.group() == "webkitpy":
+            self._test_results_reader = WebkitpyTestResultsReader(self._tool, self._port.python_unittest_results_directory())
         else:
             self._test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
 
index ef91f0b..efdbf7b 100644 (file)
@@ -30,7 +30,8 @@
 from webkitpy.thirdparty.mock import Mock
 from webkitpy.common.host import Host
 from webkitpy.common.host_mock import MockHost
-from webkitpy.common.net.bindingstestresults import BindingsTestResults
+from webkitpy.common.net.generictestresults import BindingsTestResults
+from webkitpy.common.net.generictestresults import WebkitpyTestResults
 from webkitpy.common.net.jsctestresults import JSCTestResults
 from webkitpy.common.net.layouttestresults import LayoutTestResults
 from webkitpy.common.system.outputcapture import OutputCapture
@@ -63,6 +64,12 @@ class TestBindingsEWS(AbstractEarlyWarningSystem):
     _group = "bindings"
 
 
+class TestWebkitpyEWS(AbstractEarlyWarningSystem):
+    port_name = "mac"
+    _build_style = None
+    _group = "webkitpy"
+
+
 class AbstractEarlyWarningSystemTest(QueuesTest):
     def _test_message(self, ews, results, message):
         ews.bind_to_tool(MockTool())
@@ -94,6 +101,12 @@ class AbstractEarlyWarningSystemTest(QueuesTest):
         message = "New failing tests:\n(JS) TestMapLike.idl\n(JS) TestNode.idl"
         self._test_message(ews, results, message)
 
+    def test_failing_webkitpy_tests_message(self):
+        ews = TestWebkitpyEWS()
+        results = lambda a: WebkitpyTestResults(["webkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"])
+        message = "New failing tests:\nwebkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"
+        self._test_message(ews, results, message)
+
 
 class MockEarlyWarningSystemTaskForInconclusiveJSCResults(EarlyWarningSystemTask):
     def _test_patch(self):
@@ -179,16 +192,17 @@ MOCK: update_status: %(name)s Checked relevance of patch
     def test_ews_name(self):
         # These are the names EWS's infrastructure expects, check that they work
         expected_names = {
+            'bindings-ews',
             'gtk-wk2-ews',
-            'win-ews',
             'ios-ews',
             'ios-sim-ews',
+            'jsc-ews',
+            'mac-32bit-ews',
+            'mac-debug-ews',
             'mac-ews',
             'mac-wk2-ews',
-            'mac-debug-ews',
-            'mac-32bit-ews',
-            'bindings-ews',
-            'jsc-ews',
+            'webkitpy-ews',
+            'win-ews',
             'wpe-ews',
         }
         classes = AbstractEarlyWarningSystem.load_ews_classes()
index ce761a3..49f9529 100644 (file)
@@ -64,9 +64,15 @@ class CheckPatchRelevance(AbstractStep):
         "Tools/Scripts/webkitdirs.pm",
     ]
 
+    webkitpy_paths = [
+        "Tools/Scripts/webkitpy/",
+        "Tools/QueueStatusServer/",
+    ]
+
     group_to_paths_mapping = {
         'bindings': bindings_paths,
         'jsc': jsc_paths,
+        'webkitpy': webkitpy_paths,
     }
 
     def _changes_are_relevant(self, changed_files):
index c8359d2..f45da55 100644 (file)
@@ -70,6 +70,10 @@ class RunTests(AbstractStep):
             self._run_bindings_tests()
             return
 
+        if self._options.group == "webkitpy":
+            self._run_webkitpy_tests()
+            return
+
         if self._options.iterate_on_new_tests:
             _log.info("Running run-webkit-tests on new tests")
             self._run_webkit_tests(self._options.iterate_on_new_tests)
@@ -165,6 +169,14 @@ class RunTests(AbstractStep):
         args.append("--json-output=%s" % results_file_path)
         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
 
+    def _run_webkitpy_tests(self):
+        args = self._tool.deprecated_port().run_python_unittests_command()
+        results_directory = self._tool.port_factory.get(options=self._options).python_unittest_results_directory()
+        self._tool.filesystem.maybe_make_directory(results_directory)
+        results_file_path = self._tool.filesystem.join(results_directory, "webkitpy_test_results.json")
+        args.append("--json-output={}".format(results_file_path))
+        self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
+
     def _run_api_tests(self):
         args = self._tool.deprecated_port().run_api_tests_command(self._options.build_style)
         results_directory = self._tool.port_factory.get(options=self._options).api_results_directory()
index 6fc41ea..1d280a3 100644 (file)
@@ -261,6 +261,19 @@ This patch does not have relevant changes.
 """
         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 
+    def test_runtests_webkitpy(self):
+        mock_options = self._step_options()
+        mock_options.non_interactive = False
+        mock_options.group = "webkitpy"
+        step = steps.RunTests(MockTool(log_executive=True), mock_options)
+        tool = MockTool(log_executive=True)
+        # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
+        tool._deprecated_port = DeprecatedPort()
+        step = steps.RunTests(tool, mock_options)
+        expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy', '--json-output=/tmp/python-unittest-results/webkitpy_test_results.json'], cwd=/mock-checkout
+"""
+        OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
+
     def test_patch_relevant_bindings(self):
         self.maxDiff = None
         mock_options = self._step_options()