91704df559899d6a1ac769d1f252e584c9b68d03
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem_unittest.py
1 # Copyright (C) 2009 Google Inc. All rights reserved.
2 # Copyright (C) 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #    * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #    * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #    * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 from webkitpy.thirdparty.mock import Mock
31 from webkitpy.common.host import Host
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.net.generictestresults import BindingsTestResults
34 from webkitpy.common.net.generictestresults import WebkitpyTestResults
35 from webkitpy.common.net.jsctestresults import JSCTestResults
36 from webkitpy.common.net.layouttestresults import LayoutTestResults
37 from webkitpy.common.system.outputcapture import OutputCapture
38 from webkitpy.layout_tests.models import test_results
39 from webkitpy.layout_tests.models import test_failures
40 from webkitpy.port.factory import PortFactory
41 from webkitpy.tool.bot.queueengine import QueueEngine
42 from webkitpy.tool.commands.earlywarningsystem import *
43 from webkitpy.tool.commands.queues import PatchProcessingQueue
44 from webkitpy.tool.commands.queuestest import QueuesTest
45 from webkitpy.tool.mocktool import MockTool, MockOptions
46
47
48 # Needed to define port_name, used in AbstractEarlyWarningSystem.__init__
49 class TestEWS(AbstractEarlyWarningSystem):
50     port_name = "win"  # Needs to be a port which port/factory understands.
51     _build_style = None
52     _group = None
53
54
55 class TestJSCEWS(AbstractEarlyWarningSystem):
56     port_name = "mac"  # Needs to be a port which port/factory understands.
57     _build_style = None
58     _group = "jsc"
59
60
61 class TestBindingsEWS(AbstractEarlyWarningSystem):
62     port_name = "mac"
63     _build_style = None
64     _group = "bindings"
65
66
67 class TestWebkitpyEWS(AbstractEarlyWarningSystem):
68     port_name = "mac"
69     _build_style = None
70     _group = "webkitpy"
71
72
73 class AbstractEarlyWarningSystemTest(QueuesTest):
74     def _test_message(self, ews, results, message):
75         ews.bind_to_tool(MockTool())
76         ews.host = MockHost()
77         ews._options = MockOptions(port=None, confirm=False)
78         OutputCapture().assert_outputs(self, ews.begin_work_queue, expected_logs=self._default_begin_work_queue_logs(ews.name))
79         task = Mock()
80         task.results_from_patch_test_run = results
81         patch = ews._tool.bugs.fetch_attachment(10000)
82         self.assertMultiLineEqual(ews._failing_tests_message(task, patch), message)
83
84     def test_failing_tests_message(self):
85         ews = TestEWS()
86         results = lambda a: LayoutTestResults([test_results.TestResult("foo.html", failures=[test_failures.FailureTextMismatch()]),
87                                                 test_results.TestResult("bar.html", failures=[test_failures.FailureTextMismatch()])],
88                                                 did_exceed_test_failure_limit=False)
89         message = "New failing tests:\nfoo.html\nbar.html"
90         self._test_message(ews, results, message)
91
92     def test_failing_jsc_tests_message(self):
93         ews = TestJSCEWS()
94         results = lambda a: JSCTestResults(False, ["es6.yaml/es6/typed_arrays_Int8Array.js.default", "es6.yaml/es6/typed_arrays_Uint8Array.js.default"])
95         message = "New failing tests:\nes6.yaml/es6/typed_arrays_Int8Array.js.default\nes6.yaml/es6/typed_arrays_Uint8Array.js.default\napiTests"
96         self._test_message(ews, results, message)
97
98     def test_failing_bindings_tests_message(self):
99         ews = TestBindingsEWS()
100         results = lambda a: BindingsTestResults(["(JS) TestMapLike.idl", "(JS) TestNode.idl"])
101         message = "New failing tests:\n(JS) TestMapLike.idl\n(JS) TestNode.idl"
102         self._test_message(ews, results, message)
103
104     def test_failing_webkitpy_tests_message(self):
105         ews = TestWebkitpyEWS()
106         results = lambda a: WebkitpyTestResults(["webkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"])
107         message = "New failing tests:\nwebkitpy.tool.commands.earlywarningsystem_unittest.EarlyWarningSystemTest.test_ews_name"
108         self._test_message(ews, results, message)
109
110
111 class MockEarlyWarningSystemTaskForInconclusiveJSCResults(EarlyWarningSystemTask):
112     def _test_patch(self):
113         self._test()
114         results = self._delegate.test_results()
115         return bool(results)
116
117
118 class MockAbstractEarlyWarningSystemForInconclusiveJSCResults(AbstractEarlyWarningSystem):
119     def _create_task(self, patch):
120         task = MockEarlyWarningSystemTaskForInconclusiveJSCResults(self, patch, self._options.run_tests)
121         return task
122
123
124 class EarlyWarningSystemTest(QueuesTest):
125     def _default_expected_logs(self, ews, conclusive, work_item, will_fetch_from_status_server=False):
126         string_replacements = {
127             "name": ews.name,
128             "port": ews.port_name,
129             "architecture": " --architecture=%s" % ews.architecture if ews.architecture else "",
130             "build_style": ews.build_style(),
131             "group": ews.group(),
132             'patch_id': work_item.id() if work_item else QueuesTest.mock_work_item.id(),
133         }
134
135         if will_fetch_from_status_server:
136             status_server_fetch_line = 'MOCK: fetch_attachment: %(patch_id)s\n' % string_replacements
137         else:
138             status_server_fetch_line = ''
139         string_replacements['status_server_fetch_line'] = status_server_fetch_line
140
141         if ews.should_build:
142             build_line = "%(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\nMOCK: update_status: %(name)s Built patch\n" % string_replacements
143         else:
144             build_line = ""
145         string_replacements['build_line'] = build_line
146
147         if ews.run_tests:
148             run_tests_line = "%(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com build-and-test --no-clean --no-update --test --non-interactive --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\nMOCK: update_status: %(name)s Passed tests\n" % string_replacements
149         else:
150             run_tests_line = ""
151         string_replacements['run_tests_line'] = run_tests_line
152
153         if conclusive:
154             result_lines = "MOCK: update_status: %(name)s Pass\nMOCK: release_work_item: %(name)s %(patch_id)s\n" % string_replacements
155         else:
156             result_lines = "MOCK: release_lock: %(name)s %(patch_id)s\n" % string_replacements
157         string_replacements['result_lines'] = result_lines
158
159         expected_logs = {
160             "begin_work_queue": self._default_begin_work_queue_logs(ews.name),
161             "process_work_item": """MOCK: update_status: %(name)s Started processing patch
162 %(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com clean --port=%(port)s%(architecture)s
163 MOCK: update_status: %(name)s Cleaned working directory
164 %(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com update --port=%(port)s%(architecture)s
165 MOCK: update_status: %(name)s Updated working directory
166 %(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com apply-attachment --no-update --non-interactive %(patch_id)s --port=%(port)s%(architecture)s
167 MOCK: update_status: %(name)s Applied patch
168 %(status_server_fetch_line)sRunning: webkit-patch --status-host=example.com check-patch-relevance --quiet --group=%(group)s --port=%(port)s%(architecture)s
169 MOCK: update_status: %(name)s Checked relevance of patch
170 %(build_line)s%(run_tests_line)s%(result_lines)s""" % string_replacements,
171             "handle_unexpected_error": "Mock error message\n",
172             "handle_script_error": "ScriptError error message\n\nMOCK output\n",
173         }
174         return expected_logs
175
176     def _test_ews(self, ews, results_are_conclusive=True, use_security_sensitive_patch=False):
177         ews.bind_to_tool(MockTool())
178         ews.host = MockHost()
179         options = Mock()
180         options.port = None
181         options.run_tests = ews.run_tests
182         work_item = MockTool().bugs.fetch_attachment(10008) if use_security_sensitive_patch else None
183         self.assert_queue_outputs(ews, work_item=work_item, expected_logs=self._default_expected_logs(ews, results_are_conclusive, work_item, will_fetch_from_status_server=bool(use_security_sensitive_patch)), options=options)
184
185     def test_ewses(self):
186         classes = AbstractEarlyWarningSystem.load_ews_classes()
187         self.assertTrue(classes)
188         self.maxDiff = None
189         for ews_class in classes:
190             self._test_ews(ews_class())
191
192     def test_ewses_with_security_sensitive_patch(self):
193         classes = AbstractEarlyWarningSystem.load_ews_classes()
194         self.assertTrue(classes)
195         self.maxDiff = None
196         for ews_class in classes:
197             self._test_ews(ews_class(), use_security_sensitive_patch=True)
198
199     def test_inconclusive_jsc_test_results(self):
200         classes = MockAbstractEarlyWarningSystemForInconclusiveJSCResults.load_ews_classes()
201         self.assertTrue(classes)
202         self.maxDiff = None
203         test_classes = filter(lambda x: x.run_tests and x.group == "jsc", classes)
204         for ews_class in test_classes:
205             self._test_ews(ews_class(), False)
206
207     def test_ews_name(self):
208         # These are the names EWS's infrastructure expects, check that they work
209         expected_names = {
210             'bindings-ews',
211             'gtk-wk2-ews',
212             'ios-ews',
213             'ios-sim-ews',
214             'jsc-ews',
215             'jsc-mips-ews',
216             'jsc-armv7-ews',
217             'mac-32bit-ews',
218             'mac-debug-ews',
219             'mac-ews',
220             'mac-wk2-ews',
221             'webkitpy-ews',
222             'win-ews',
223             'wpe-ews',
224             'wincairo-ews',
225         }
226         classes = AbstractEarlyWarningSystem.load_ews_classes()
227         names = {cls.name for cls in classes}
228         missing_names = expected_names - names
229         unexpected_names = names - expected_names
230         if missing_names:
231             raise AssertionError("'{}' is not a valid EWS command but is used by EWS's infrastructure".format(missing_names.pop()))
232         if unexpected_names:
233             raise AssertionError("'{}' is a valid EWS command but is not used by EWS's infrastructure".format(unexpected_names.pop()))