From 6e5fe1d3b458f70bec7c1b4872b821192e2306b6 Mon Sep 17 00:00:00 2001 From: "aakash_jain@apple.com" Date: Tue, 7 Nov 2017 22:30:57 +0000 Subject: [PATCH] Add support for webkitpy tests EWS https://bugs.webkit.org/show_bug.cgi?id=178480 Reviewed by Alexey Proskuryakov. * Scripts/webkitpy/common/config/ews.json: Added webkitpy EWS. * Scripts/webkitpy/common/net/generictestresults.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults.py. * Scripts/webkitpy/common/net/generictestresults_unittest.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults_unittest.py. * Scripts/webkitpy/tool/bot/patchanalysistask.py: (PatchAnalysisTask._test_patch): Do not retry webkitpy test since they are not flaky. * Scripts/webkitpy/tool/bot/bindingstestresultsreader.py: Updated import for BindingsTestResult. * Scripts/webkitpy/tool/bot/retrylogic_unittest.py: Ditto. * Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py: Copied from Scripts/webkitpy/tool/bot/bindingstestresultsreader.py. * Scripts/webkitpy/tool/commands/earlywarningsystem.py: (AbstractEarlyWarningSystem.begin_work_queue): Added webkitpy test result reader. * Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py: Updated unit-tests. * Scripts/webkitpy/tool/steps/checkpatchrelevance.py: Run webkitpy EWS only for relevant changes. * Scripts/webkitpy/tool/steps/runtests.py: (RunTests.run): Run webkitpy tests when --group=webkitpy is passed. (RunTests._run_webkitpy_tests): Method to invoke webkitpy tests. * Scripts/webkitpy/tool/steps/steps_unittest.py: (test_runtests_webkitpy): Added unit-test. git-svn-id: https://svn.webkit.org/repository/webkit/trunk@224547 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- Tools/ChangeLog | 25 ++++++++++++++++ Tools/Scripts/webkitpy/common/config/ews.json | 7 +++++ ...indingstestresults.py => generictestresults.py} | 10 ++++++- ..._unittest.py => generictestresults_unittest.py} | 14 ++++----- .../webkitpy/tool/bot/bindingstestresultsreader.py | 2 +- .../Scripts/webkitpy/tool/bot/patchanalysistask.py | 2 ++ .../webkitpy/tool/bot/retrylogic_unittest.py | 2 +- .../webkitpy/tool/bot/webkitpytestresultsreader.py | 35 ++++++++++++++++++++++ .../webkitpy/tool/commands/earlywarningsystem.py | 3 ++ .../tool/commands/earlywarningsystem_unittest.py | 26 ++++++++++++---- .../webkitpy/tool/steps/checkpatchrelevance.py | 6 ++++ Tools/Scripts/webkitpy/tool/steps/runtests.py | 12 ++++++++ .../Scripts/webkitpy/tool/steps/steps_unittest.py | 13 ++++++++ 13 files changed, 141 insertions(+), 16 deletions(-) rename Tools/Scripts/webkitpy/common/net/{bindingstestresults.py => generictestresults.py} (93%) rename Tools/Scripts/webkitpy/common/net/{bindingstestresults_unittest.py => generictestresults_unittest.py} (75%) create mode 100644 Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py diff --git a/Tools/ChangeLog b/Tools/ChangeLog index ed45ae0..3eaa0df 100644 --- a/Tools/ChangeLog +++ b/Tools/ChangeLog @@ -1,3 +1,28 @@ +2017-11-07 Aakash Jain + + Add support for webkitpy tests EWS + https://bugs.webkit.org/show_bug.cgi?id=178480 + + Reviewed by Alexey Proskuryakov. + + * Scripts/webkitpy/common/config/ews.json: Added webkitpy EWS. + * Scripts/webkitpy/common/net/generictestresults.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults.py. + * Scripts/webkitpy/common/net/generictestresults_unittest.py: Renamed from Scripts/webkitpy/common/net/bindingstestresults_unittest.py. + * Scripts/webkitpy/tool/bot/patchanalysistask.py: + (PatchAnalysisTask._test_patch): Do not retry webkitpy test since they are not flaky. + * Scripts/webkitpy/tool/bot/bindingstestresultsreader.py: Updated import for BindingsTestResult. + * Scripts/webkitpy/tool/bot/retrylogic_unittest.py: Ditto. + * Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py: Copied from Scripts/webkitpy/tool/bot/bindingstestresultsreader.py. + * Scripts/webkitpy/tool/commands/earlywarningsystem.py: + (AbstractEarlyWarningSystem.begin_work_queue): Added webkitpy test result reader. + * Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py: Updated unit-tests. + * Scripts/webkitpy/tool/steps/checkpatchrelevance.py: Run webkitpy EWS only for relevant changes. + * Scripts/webkitpy/tool/steps/runtests.py: + (RunTests.run): Run webkitpy tests when --group=webkitpy is passed. + (RunTests._run_webkitpy_tests): Method to invoke webkitpy tests. + * Scripts/webkitpy/tool/steps/steps_unittest.py: + (test_runtests_webkitpy): Added unit-test. + 2017-11-07 Commit Queue Unreviewed, rolling out r224512 and r224521. diff --git a/Tools/Scripts/webkitpy/common/config/ews.json b/Tools/Scripts/webkitpy/common/config/ews.json index cea9947..342a6f8 100644 --- a/Tools/Scripts/webkitpy/common/config/ews.json +++ b/Tools/Scripts/webkitpy/common/config/ews.json @@ -59,6 +59,13 @@ "runTests": true, "shouldBuild": false }, + "webkitpy EWS": { + "port": "mac", + "name": "webkitpy-ews", + "group": "webkitpy", + "runTests": true, + "shouldBuild": false + }, "WPE EWS": { "port": "wpe", "watchers": [ diff --git a/Tools/Scripts/webkitpy/common/net/bindingstestresults.py b/Tools/Scripts/webkitpy/common/net/generictestresults.py similarity index 93% rename from Tools/Scripts/webkitpy/common/net/bindingstestresults.py rename to Tools/Scripts/webkitpy/common/net/generictestresults.py index e0f8527..fbaf706 100644 --- a/Tools/Scripts/webkitpy/common/net/bindingstestresults.py +++ b/Tools/Scripts/webkitpy/common/net/generictestresults.py @@ -27,7 +27,7 @@ from webkitpy.common.net.abstracttestresults import AbstractTestResults _log = logging.getLogger(__name__) -class BindingsTestResults(AbstractTestResults): +class GenericTestResults(AbstractTestResults): def __init__(self, failures): self._failures = failures @@ -57,3 +57,11 @@ class BindingsTestResults(AbstractTestResults): # No defined failure limit for bindings tests. def did_exceed_test_failure_limit(self): return False + + +class BindingsTestResults(GenericTestResults): + pass + + +class WebkitpyTestResults(GenericTestResults): + pass diff --git a/Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py b/Tools/Scripts/webkitpy/common/net/generictestresults_unittest.py similarity index 75% rename from Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py rename to Tools/Scripts/webkitpy/common/net/generictestresults_unittest.py index cb16805..9171963 100644 --- a/Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py +++ b/Tools/Scripts/webkitpy/common/net/generictestresults_unittest.py @@ -22,20 +22,20 @@ import unittest -from webkitpy.common.net.bindingstestresults import BindingsTestResults +from webkitpy.common.net.generictestresults import GenericTestResults -class BindingsTestResultsTest(unittest.TestCase): +class GenericTestResultsTest(unittest.TestCase): def test_results_from_string(self): incomplete_json = '{"key2": []}' - self.assertEqual(None, BindingsTestResults.results_from_string(incomplete_json)) + self.assertEqual(None, GenericTestResults.results_from_string(incomplete_json)) def test_results_from_string_success(self): no_failures_string = '{"failures": []}' - no_failures_results = BindingsTestResults([]) - self.assertTrue(no_failures_results.equals(BindingsTestResults.results_from_string(no_failures_string))) + no_failures_results = GenericTestResults([]) + self.assertTrue(no_failures_results.equals(GenericTestResults.results_from_string(no_failures_string))) self.assertTrue(no_failures_results.all_passed()) test_string = '{"failures": ["failure1"]}' - test_results = BindingsTestResults(["failure1"]) - self.assertTrue(test_results.equals(BindingsTestResults.results_from_string(test_string))) + test_results = GenericTestResults(["failure1"]) + self.assertTrue(test_results.equals(GenericTestResults.results_from_string(test_string))) diff --git a/Tools/Scripts/webkitpy/tool/bot/bindingstestresultsreader.py b/Tools/Scripts/webkitpy/tool/bot/bindingstestresultsreader.py index 8fca2fd..525528b 100644 --- a/Tools/Scripts/webkitpy/tool/bot/bindingstestresultsreader.py +++ b/Tools/Scripts/webkitpy/tool/bot/bindingstestresultsreader.py @@ -22,7 +22,7 @@ import logging -from webkitpy.common.net.bindingstestresults import BindingsTestResults +from webkitpy.common.net.generictestresults import BindingsTestResults from webkitpy.tool.bot.abstracttestresultsreader import AbstractTestResultsReader _log = logging.getLogger(__name__) diff --git a/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py b/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py index 5de8386..3ab2429 100644 --- a/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py +++ b/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py @@ -368,6 +368,8 @@ class PatchAnalysisTask(object): return self._retry_jsc_tests() elif hasattr(self._delegate, 'group') and self._delegate.group() == "bindings": return self._retry_bindings_tests() + elif hasattr(self._delegate, 'group') and self._delegate.group() == "webkitpy": + return False else: return self._retry_layout_tests() diff --git a/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py b/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py index 4c1b440..9a72c7c 100644 --- a/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py +++ b/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py @@ -23,7 +23,7 @@ import logging import unittest -from webkitpy.common.net.bindingstestresults import BindingsTestResults +from webkitpy.common.net.generictestresults import BindingsTestResults from webkitpy.common.net.jsctestresults import JSCTestResults from webkitpy.common.system.executive import ScriptError from webkitpy.tool.bot.patchanalysistask import * diff --git a/Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py b/Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py new file mode 100644 index 0000000..852ba85 --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/webkitpytestresultsreader.py @@ -0,0 +1,35 @@ +# Copyright (C) 2017 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +from webkitpy.common.net.generictestresults import WebkitpyTestResults +from webkitpy.tool.bot.abstracttestresultsreader import AbstractTestResultsReader + +_log = logging.getLogger(__name__) + + +class WebkitpyTestResultsReader(AbstractTestResultsReader): + def results(self): + results_path = self._host.filesystem.join(self._results_directory, 'webkitpy_test_results.json') + contents = self._read_file_contents(results_path) + return WebkitpyTestResults.results_from_string(contents) diff --git a/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py b/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py index e82b0a7..54a3a98 100644 --- a/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py +++ b/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py @@ -39,6 +39,7 @@ from webkitpy.common.system.executive import ScriptError from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate from webkitpy.tool.bot.bindingstestresultsreader import BindingsTestResultsReader from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader +from webkitpy.tool.bot.webkitpytestresultsreader import WebkitpyTestResultsReader from webkitpy.tool.bot.jsctestresultsreader import JSCTestResultsReader from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch, PatchIsNotValid, PatchIsNotApplicable from webkitpy.tool.bot.queueengine import QueueEngine @@ -62,6 +63,8 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDele self._test_results_reader = JSCTestResultsReader(self._tool, self._port.jsc_results_directory()) elif self.group() == "bindings": self._test_results_reader = BindingsTestResultsReader(self._tool, self._port.jsc_results_directory()) + elif self.group() == "webkitpy": + self._test_results_reader = WebkitpyTestResultsReader(self._tool, self._port.python_unittest_results_directory()) else: self._test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory()) diff --git a/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py b/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py index ef91f0b..efdbf7b 100644 --- a/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py @@ -30,7 +30,8 @@ from webkitpy.thirdparty.mock import Mock from webkitpy.common.host import Host from webkitpy.common.host_mock import MockHost -from webkitpy.common.net.bindingstestresults import BindingsTestResults +from webkitpy.common.net.generictestresults import BindingsTestResults +from webkitpy.common.net.generictestresults import WebkitpyTestResults from webkitpy.common.net.jsctestresults import JSCTestResults from webkitpy.common.net.layouttestresults import LayoutTestResults from webkitpy.common.system.outputcapture import OutputCapture @@ -63,6 +64,12 @@ class TestBindingsEWS(AbstractEarlyWarningSystem): _group = "bindings" +class TestWebkitpyEWS(AbstractEarlyWarningSystem): + port_name = "mac" + _build_style = None + _group = "webkitpy" + + class AbstractEarlyWarningSystemTest(QueuesTest): def _test_message(self, ews, results, message): ews.bind_to_tool(MockTool()) @@ -94,6 +101,12 @@ class AbstractEarlyWarningSystemTest(QueuesTest): message = "New failing tests:\n(JS) TestMapLike.idl\n(JS) TestNode.idl" self._test_message(ews, results, message) + def test_failing_webkitpy_tests_message(self): + ews = TestWebkitpyEWS() + results = lambda a: WebkitpyTestResults(["webkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"]) + message = "New failing tests:\nwebkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name" + self._test_message(ews, results, message) + class MockEarlyWarningSystemTaskForInconclusiveJSCResults(EarlyWarningSystemTask): def _test_patch(self): @@ -179,16 +192,17 @@ MOCK: update_status: %(name)s Checked relevance of patch def test_ews_name(self): # These are the names EWS's infrastructure expects, check that they work expected_names = { + 'bindings-ews', 'gtk-wk2-ews', - 'win-ews', 'ios-ews', 'ios-sim-ews', + 'jsc-ews', + 'mac-32bit-ews', + 'mac-debug-ews', 'mac-ews', 'mac-wk2-ews', - 'mac-debug-ews', - 'mac-32bit-ews', - 'bindings-ews', - 'jsc-ews', + 'webkitpy-ews', + 'win-ews', 'wpe-ews', } classes = AbstractEarlyWarningSystem.load_ews_classes() diff --git a/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py b/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py index ce761a3..49f9529 100644 --- a/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py +++ b/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py @@ -64,9 +64,15 @@ class CheckPatchRelevance(AbstractStep): "Tools/Scripts/webkitdirs.pm", ] + webkitpy_paths = [ + "Tools/Scripts/webkitpy/", + "Tools/QueueStatusServer/", + ] + group_to_paths_mapping = { 'bindings': bindings_paths, 'jsc': jsc_paths, + 'webkitpy': webkitpy_paths, } def _changes_are_relevant(self, changed_files): diff --git a/Tools/Scripts/webkitpy/tool/steps/runtests.py b/Tools/Scripts/webkitpy/tool/steps/runtests.py index c8359d2..f45da55 100644 --- a/Tools/Scripts/webkitpy/tool/steps/runtests.py +++ b/Tools/Scripts/webkitpy/tool/steps/runtests.py @@ -70,6 +70,10 @@ class RunTests(AbstractStep): self._run_bindings_tests() return + if self._options.group == "webkitpy": + self._run_webkitpy_tests() + return + if self._options.iterate_on_new_tests: _log.info("Running run-webkit-tests on new tests") self._run_webkit_tests(self._options.iterate_on_new_tests) @@ -165,6 +169,14 @@ class RunTests(AbstractStep): args.append("--json-output=%s" % results_file_path) self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root) + def _run_webkitpy_tests(self): + args = self._tool.deprecated_port().run_python_unittests_command() + results_directory = self._tool.port_factory.get(options=self._options).python_unittest_results_directory() + self._tool.filesystem.maybe_make_directory(results_directory) + results_file_path = self._tool.filesystem.join(results_directory, "webkitpy_test_results.json") + args.append("--json-output={}".format(results_file_path)) + self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root) + def _run_api_tests(self): args = self._tool.deprecated_port().run_api_tests_command(self._options.build_style) results_directory = self._tool.port_factory.get(options=self._options).api_results_directory() diff --git a/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py b/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py index 6fc41ea..1d280a3 100644 --- a/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py +++ b/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py @@ -261,6 +261,19 @@ This patch does not have relevant changes. """ OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs) + def test_runtests_webkitpy(self): + mock_options = self._step_options() + mock_options.non_interactive = False + mock_options.group = "webkitpy" + step = steps.RunTests(MockTool(log_executive=True), mock_options) + tool = MockTool(log_executive=True) + # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment. + tool._deprecated_port = DeprecatedPort() + step = steps.RunTests(tool, mock_options) + expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy', '--json-output=/tmp/python-unittest-results/webkitpy_test_results.json'], cwd=/mock-checkout +""" + OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs) + def test_patch_relevant_bindings(self): self.maxDiff = None mock_options = self._step_options() -- 1.8.3.1