6ecaed0bb26ec394e4fbb24a3ac3f3ebcb6c81b2
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem_unittest.py
1 # Copyright (C) 2009 Google Inc. All rights reserved.
2 # Copyright (C) 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #    * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #    * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #    * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 from webkitpy.thirdparty.mock import Mock
31 from webkitpy.common.host import Host
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.net.generictestresults import BindingsTestResults
34 from webkitpy.common.net.generictestresults import WebkitpyTestResults
35 from webkitpy.common.net.jsctestresults import JSCTestResults
36 from webkitpy.common.net.layouttestresults import LayoutTestResults
37 from webkitpy.common.system.outputcapture import OutputCapture
38 from webkitpy.layout_tests.models import test_results
39 from webkitpy.layout_tests.models import test_failures
40 from webkitpy.port.factory import PortFactory
41 from webkitpy.tool.bot.queueengine import QueueEngine
42 from webkitpy.tool.commands.earlywarningsystem import *
43 from webkitpy.tool.commands.queues import PatchProcessingQueue
44 from webkitpy.tool.commands.queuestest import QueuesTest
45 from webkitpy.tool.mocktool import MockTool, MockOptions
46
47
48 # Needed to define port_name, used in AbstractEarlyWarningSystem.__init__
49 class TestEWS(AbstractEarlyWarningSystem):
50     port_name = "win"  # Needs to be a port which port/factory understands.
51     _build_style = None
52     _group = None
53
54
55 class TestJSCEWS(AbstractEarlyWarningSystem):
56     port_name = "mac"  # Needs to be a port which port/factory understands.
57     _build_style = None
58     _group = "jsc"
59
60
61 class TestBindingsEWS(AbstractEarlyWarningSystem):
62     port_name = "mac"
63     _build_style = None
64     _group = "bindings"
65
66
67 class TestWebkitpyEWS(AbstractEarlyWarningSystem):
68     port_name = "mac"
69     _build_style = None
70     _group = "webkitpy"
71
72
73 class AbstractEarlyWarningSystemTest(QueuesTest):
74     def _test_message(self, ews, results, message):
75         ews.bind_to_tool(MockTool())
76         ews.host = MockHost()
77         ews._options = MockOptions(port=None, confirm=False)
78         OutputCapture().assert_outputs(self, ews.begin_work_queue, expected_logs=self._default_begin_work_queue_logs(ews.name))
79         task = Mock()
80         task.results_from_patch_test_run = results
81         patch = ews._tool.bugs.fetch_attachment(10000)
82         self.assertMultiLineEqual(ews._failing_tests_message(task, patch), message)
83
84     def test_failing_tests_message(self):
85         ews = TestEWS()
86         results = lambda a: LayoutTestResults([test_results.TestResult("foo.html", failures=[test_failures.FailureTextMismatch()]),
87                                                 test_results.TestResult("bar.html", failures=[test_failures.FailureTextMismatch()])],
88                                                 did_exceed_test_failure_limit=False)
89         message = "New failing tests:\nfoo.html\nbar.html"
90         self._test_message(ews, results, message)
91
92     def test_failing_jsc_tests_message(self):
93         ews = TestJSCEWS()
94         results = lambda a: JSCTestResults(False, ["es6.yaml/es6/typed_arrays_Int8Array.js.default", "es6.yaml/es6/typed_arrays_Uint8Array.js.default"])
95         message = "New failing tests:\nes6.yaml/es6/typed_arrays_Int8Array.js.default\nes6.yaml/es6/typed_arrays_Uint8Array.js.default\napiTests"
96         self._test_message(ews, results, message)
97
98     def test_failing_bindings_tests_message(self):
99         ews = TestBindingsEWS()
100         results = lambda a: BindingsTestResults(["(JS) TestMapLike.idl", "(JS) TestNode.idl"])
101         message = "New failing tests:\n(JS) TestMapLike.idl\n(JS) TestNode.idl"
102         self._test_message(ews, results, message)
103
104     def test_failing_webkitpy_tests_message(self):
105         ews = TestWebkitpyEWS()
106         results = lambda a: WebkitpyTestResults(["webkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"])
107         message = "New failing tests:\nwebkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"
108         self._test_message(ews, results, message)
109
110
111 class MockEarlyWarningSystemTaskForInconclusiveJSCResults(EarlyWarningSystemTask):
112     def _test_patch(self):
113         self._test()
114         results = self._delegate.test_results()
115         return bool(results)
116
117
118 class MockAbstractEarlyWarningSystemForInconclusiveJSCResults(AbstractEarlyWarningSystem):
119     def _create_task(self, patch):
120         task = MockEarlyWarningSystemTaskForInconclusiveJSCResults(self, patch, self._options.run_tests)
121         return task
122
123
124 class EarlyWarningSystemTest(QueuesTest):
125     def _default_expected_logs(self, ews, conclusive):
126         string_replacements = {
127             "name": ews.name,
128             "port": ews.port_name,
129             "architecture": " --architecture=%s" % ews.architecture if ews.architecture else "",
130             "build_style": ews.build_style(),
131             "group": ews.group(),
132         }
133
134         if ews.should_build:
135             build_line = "Running: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\nMOCK: update_status: %(name)s Built patch\n" % string_replacements
136         else:
137             build_line = ""
138         string_replacements['build_line'] = build_line
139
140         if ews.run_tests:
141             run_tests_line = "Running: webkit-patch --status-host=example.com build-and-test --no-clean --no-update --test --non-interactive --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\nMOCK: update_status: %(name)s Passed tests\n" % string_replacements
142         else:
143             run_tests_line = ""
144         string_replacements['run_tests_line'] = run_tests_line
145
146         if conclusive:
147             result_lines = "MOCK: update_status: %(name)s Pass\nMOCK: release_work_item: %(name)s 10000\n" % string_replacements
148         else:
149             result_lines = "MOCK: release_lock: %(name)s 10000\n" % string_replacements
150         string_replacements['result_lines'] = result_lines
151
152         expected_logs = {
153             "begin_work_queue": self._default_begin_work_queue_logs(ews.name),
154             "process_work_item": """MOCK: update_status: %(name)s Started processing patch
155 Running: webkit-patch --status-host=example.com clean --port=%(port)s%(architecture)s
156 MOCK: update_status: %(name)s Cleaned working directory
157 Running: webkit-patch --status-host=example.com update --port=%(port)s%(architecture)s
158 MOCK: update_status: %(name)s Updated working directory
159 Running: webkit-patch --status-host=example.com apply-attachment --no-update --non-interactive 10000 --port=%(port)s%(architecture)s
160 MOCK: update_status: %(name)s Applied patch
161 Running: webkit-patch --status-host=example.com check-patch-relevance --quiet --group=%(group)s --port=%(port)s%(architecture)s
162 MOCK: update_status: %(name)s Checked relevance of patch
163 %(build_line)s%(run_tests_line)s%(result_lines)s""" % string_replacements,
164             "handle_unexpected_error": "Mock error message\n",
165             "handle_script_error": "ScriptError error message\n\nMOCK output\n",
166         }
167         return expected_logs
168
169     def _test_ews(self, ews, results_are_conclusive=True):
170         ews.bind_to_tool(MockTool())
171         ews.host = MockHost()
172         options = Mock()
173         options.port = None
174         options.run_tests = ews.run_tests
175         self.assert_queue_outputs(ews, expected_logs=self._default_expected_logs(ews, results_are_conclusive), options=options)
176
177     def test_ewses(self):
178         classes = AbstractEarlyWarningSystem.load_ews_classes()
179         self.assertTrue(classes)
180         self.maxDiff = None
181         for ews_class in classes:
182             self._test_ews(ews_class())
183
184     def test_inconclusive_jsc_test_results(self):
185         classes = MockAbstractEarlyWarningSystemForInconclusiveJSCResults.load_ews_classes()
186         self.assertTrue(classes)
187         self.maxDiff = None
188         test_classes = filter(lambda x: x.run_tests and x.group == "jsc", classes)
189         for ews_class in test_classes:
190             self._test_ews(ews_class(), False)
191
192     def test_ews_name(self):
193         # These are the names EWS's infrastructure expects, check that they work
194         expected_names = {
195             'bindings-ews',
196             'gtk-wk2-ews',
197             'ios-ews',
198             'ios-sim-ews',
199             'jsc-ews',
200             'mac-32bit-ews',
201             'mac-debug-ews',
202             'mac-ews',
203             'mac-wk2-ews',
204             'webkitpy-ews',
205             'win-ews',
206             'wpe-ews',
207             'wincairo-ews',
208         }
209         classes = AbstractEarlyWarningSystem.load_ews_classes()
210         names = {cls.name for cls in classes}
211         missing_names = expected_names - names
212         unexpected_names = names - expected_names
213         if missing_names:
214             raise AssertionError("'{}' is not a valid EWS command but is used by EWS's infrastructure".format(missing_names.pop()))
215         if unexpected_names:
216             raise AssertionError("'{}' is a valid EWS command but is not used by EWS's infrastructure".format(unexpected_names.pop()))