Add support for webkitpy tests EWS
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / steps / steps_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.system.executive import ScriptError
33 from webkitpy.common.system.outputcapture import OutputCapture
34 from webkitpy.common.config.ports import DeprecatedPort
35 from webkitpy.tool.mocktool import MockOptions, MockTool
36
37 from webkitpy.tool import steps
38
39
40 class StepsTest(unittest.TestCase):
41
42     def setUp(self):
43         # Port._build_path() calls Tools/Scripts/webkit-build-directory and caches the result. When capturing output,
44         # this can cause the first invocation of Port._build_path() to have more output than subsequent invocations.
45         # This may cause test flakiness when test order changes. By explicitly calling Port._build_path() before running
46         # tests in this suite, we avoid such flakiness.
47         MockTool().port_factory.get(options=self._step_options())._build_path()
48
49     def _step_options(self):
50         options = MockOptions()
51         options.group = None
52         options.non_interactive = True
53         options.port = 'MOCK port'
54         options.quiet = True
55         options.test = True
56         options.iterate_on_new_tests = 0
57         options.root = '/tmp'
58         return options
59
60     def _run_step(self, step, tool=None, options=None, state=None):
61         if not tool:
62             tool = MockTool()
63         if not options:
64             options = self._step_options()
65         if not state:
66             state = {}
67         step(tool, options).run(state)
68
69     def test_update_step(self):
70         tool = MockTool()
71         options = self._step_options()
72         options.update = True
73         expected_logs = "Updating working directory\n"
74         OutputCapture().assert_outputs(self, self._run_step, [steps.Update, tool, options], expected_logs=expected_logs)
75
76     def test_prompt_for_bug_or_title_step(self):
77         tool = MockTool()
78         tool.user.prompt = lambda message: 50000
79         self._run_step(steps.PromptForBugOrTitle, tool=tool)
80
81     def _post_diff_options(self):
82         options = self._step_options()
83         options.git_commit = None
84         options.description = None
85         options.comment = None
86         options.review = True
87         options.request_commit = False
88         options.open_bug = True
89         return options
90
91     def _assert_step_output_with_bug(self, step, bug_id, expected_logs, options=None):
92         state = {'bug_id': bug_id}
93         OutputCapture().assert_outputs(self, self._run_step, [step, MockTool(), options, state], expected_logs=expected_logs)
94
95     def _assert_post_diff_output_for_bug(self, step, bug_id, expected_logs):
96         self._assert_step_output_with_bug(step, bug_id, expected_logs, self._post_diff_options())
97
98     def test_post_diff(self):
99         expected_logs = "MOCK add_patch_to_bug: bug_id=78, description=Patch, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False\nMOCK: user.open_url: http://example.com/78\n"
100         self._assert_post_diff_output_for_bug(steps.PostDiff, 78, expected_logs)
101
102     def test_post_diff_for_commit(self):
103         expected_logs = "MOCK add_patch_to_bug: bug_id=78, description=Patch for landing, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=True\n"
104         self._assert_post_diff_output_for_bug(steps.PostDiffForCommit, 78, expected_logs)
105
106     def test_ensure_bug_is_open_and_assigned(self):
107         expected_logs = "MOCK reopen_bug 50004 with comment 'Reopening to attach new patch.'\n"
108         self._assert_step_output_with_bug(steps.EnsureBugIsOpenAndAssigned, 50004, expected_logs)
109         expected_logs = "MOCK reassign_bug: bug_id=50002, assignee=None\n"
110         self._assert_step_output_with_bug(steps.EnsureBugIsOpenAndAssigned, 50002, expected_logs)
111
112     def test_runtests_args(self):
113         mock_options = self._step_options()
114         mock_options.non_interactive = False
115         mock_options.build_style = "release"
116         step = steps.RunTests(MockTool(log_executive=True), mock_options)
117         tool = MockTool(log_executive=True)
118         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
119         tool._deprecated_port = DeprecatedPort()
120         step = steps.RunTests(tool, mock_options)
121         expected_logs = """Running Python unit tests
122 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy'], cwd=/mock-checkout
123 Running Perl unit tests
124 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitperl'], cwd=/mock-checkout
125 Running JavaScriptCore tests
126 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast'], cwd=/mock-checkout
127 Running run-webkit-tests
128 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--release', '--quiet'], cwd=/mock-checkout
129 """
130         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
131
132     def test_runtests_debug_args(self):
133         mock_options = self._step_options()
134         mock_options.non_interactive = False
135         mock_options.build_style = "debug"
136         step = steps.RunTests(MockTool(log_executive=True), mock_options)
137         tool = MockTool(log_executive=True)
138         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
139         tool._deprecated_port = DeprecatedPort()
140         step = steps.RunTests(tool, mock_options)
141         expected_logs = """Running Python unit tests
142 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy'], cwd=/mock-checkout
143 Running Perl unit tests
144 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitperl'], cwd=/mock-checkout
145 Running JavaScriptCore tests
146 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast'], cwd=/mock-checkout
147 Running run-webkit-tests
148 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--debug', '--quiet'], cwd=/mock-checkout
149 """
150         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
151
152     def test_runtests_jsc(self):
153         mock_options = self._step_options()
154         mock_options.non_interactive = False
155         mock_options.build_style = "release"
156         mock_options.group = "jsc"
157         step = steps.RunTests(MockTool(log_executive=True), mock_options)
158         tool = MockTool(log_executive=True)
159         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
160         tool._deprecated_port = DeprecatedPort()
161         step = steps.RunTests(tool, mock_options)
162         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--release', '--json-output=/tmp/jsc_test_results.json'], cwd=/mock-checkout
163 """
164         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
165
166     def test_runtests_jsc_debug(self):
167         mock_options = self._step_options()
168         mock_options.non_interactive = False
169         mock_options.build_style = "debug"
170         mock_options.group = "jsc"
171         tool = MockTool(log_executive=True)
172         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
173         tool._deprecated_port = DeprecatedPort()
174         step = steps.RunTests(tool, mock_options)
175         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--debug', '--json-output=/tmp/jsc_test_results.json'], cwd=/mock-checkout
176 """
177         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
178
179     def test_build_jsc_debug(self):
180         mock_options = self._step_options()
181         mock_options.non_interactive = False
182         mock_options.build_style = "debug"
183         mock_options.build = True
184         mock_options.architecture = True
185         mock_options.group = "jsc"
186         tool = MockTool(log_executive=True)
187         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
188         tool._deprecated_port = DeprecatedPort()
189         step = steps.Build(tool, mock_options)
190         expected_logs = """Building WebKit
191 MOCK run_and_throw_if_fail: ['Tools/Scripts/build-jsc', '--debug', 'ARCHS=True'], cwd=/mock-checkout, env={'TERM': 'dumb', 'MOCK_ENVIRON_COPY': '1'}
192 """
193         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
194
195     def test_build_jsc(self):
196         mock_options = self._step_options()
197         mock_options.non_interactive = False
198         mock_options.build_style = "release"
199         mock_options.build = True
200         mock_options.architecture = True
201         mock_options.group = "jsc"
202         tool = MockTool(log_executive=True)
203         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
204         tool._deprecated_port = DeprecatedPort()
205         step = steps.Build(tool, mock_options)
206         expected_logs = """Building WebKit
207 MOCK run_and_throw_if_fail: ['Tools/Scripts/build-jsc', '--release', 'ARCHS=True'], cwd=/mock-checkout, env={'TERM': 'dumb', 'MOCK_ENVIRON_COPY': '1'}
208 """
209         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
210
211     def test_patch_relevant(self):
212         self.maxDiff = None
213         mock_options = self._step_options()
214         tool = MockTool(log_executive=True)
215         tool.scm()._mockChangedFiles = ["JSTests/MockFile1", "ChangeLog"]
216         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
217         tool._deprecated_port = DeprecatedPort()
218         step = steps.CheckPatchRelevance(tool, mock_options)
219         expected_logs = """Checking relevance of patch
220 """
221         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
222
223     def test_patch_relevant_jsc(self):
224         self.maxDiff = None
225         mock_options = self._step_options()
226         mock_options.group = "jsc"
227         tool = MockTool(log_executive=True)
228         tool.scm()._mockChangedFiles = ["JSTests/MockFile1", "ChangeLog"]
229         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
230         tool._deprecated_port = DeprecatedPort()
231         step = steps.CheckPatchRelevance(tool, mock_options)
232         expected_logs = """Checking relevance of patch
233 """
234         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
235
236     def test_patch_not_relevant_jsc(self):
237         self.maxDiff = None
238         mock_options = self._step_options()
239         mock_options.group = "jsc"
240         tool = MockTool(log_executive=True)
241         tool.scm()._mockChangedFiles = ["Tools/ChangeLog", "Tools/Scripts/webkitpy/tool/steps/steps_unittest.py"]
242         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
243         tool._deprecated_port = DeprecatedPort()
244         step = steps.CheckPatchRelevance(tool, mock_options)
245         expected_logs = """Checking relevance of patch
246 This patch does not have relevant changes.
247 """
248         with self.assertRaises(ScriptError):
249             OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
250
251     def test_runtests_bindings(self):
252         mock_options = self._step_options()
253         mock_options.non_interactive = False
254         mock_options.group = "bindings"
255         step = steps.RunTests(MockTool(log_executive=True), mock_options)
256         tool = MockTool(log_executive=True)
257         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
258         tool._deprecated_port = DeprecatedPort()
259         step = steps.RunTests(tool, mock_options)
260         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-bindings-tests', '--json-output=/tmp/bindings_test_results.json'], cwd=/mock-checkout
261 """
262         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
263
264     def test_runtests_webkitpy(self):
265         mock_options = self._step_options()
266         mock_options.non_interactive = False
267         mock_options.group = "webkitpy"
268         step = steps.RunTests(MockTool(log_executive=True), mock_options)
269         tool = MockTool(log_executive=True)
270         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
271         tool._deprecated_port = DeprecatedPort()
272         step = steps.RunTests(tool, mock_options)
273         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy', '--json-output=/tmp/python-unittest-results/webkitpy_test_results.json'], cwd=/mock-checkout
274 """
275         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
276
277     def test_patch_relevant_bindings(self):
278         self.maxDiff = None
279         mock_options = self._step_options()
280         mock_options.group = "bindings"
281         tool = MockTool(log_executive=True)
282         tool.scm()._mockChangedFiles = ["Source/WebCore/features.json", "Source/ChangeLog"]
283         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
284         tool._deprecated_port = DeprecatedPort()
285         step = steps.CheckPatchRelevance(tool, mock_options)
286         expected_logs = """Checking relevance of patch
287 """
288         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
289
290     def test_patch_not_relevant_bindings(self):
291         self.maxDiff = None
292         mock_options = self._step_options()
293         mock_options.group = "bindings"
294         tool = MockTool(log_executive=True)
295         tool.scm()._mockChangedFiles = ["Source/JavaScriptCore/Makefile", "Source/ChangeLog"]
296         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
297         tool._deprecated_port = DeprecatedPort()
298         step = steps.CheckPatchRelevance(tool, mock_options)
299         expected_logs = """Checking relevance of patch
300 This patch does not have relevant changes.
301 """
302
303     def test_runtests_api(self):
304         mock_options = self._step_options()
305         mock_options.non_interactive = False
306         mock_options.build_style = "release"
307         mock_options.group = "api"
308         step = steps.RunTests(MockTool(log_executive=True), mock_options)
309         tool = MockTool(log_executive=True)
310         tool._deprecated_port = DeprecatedPort()
311         step = steps.RunTests(tool, mock_options)
312         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-api-tests', '--release', '--json-output=/tmp/api_test_results.json'], cwd=/mock-checkout
313 """
314         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
315
316     def test_runtests_api_debug(self):
317         mock_options = self._step_options()
318         mock_options.non_interactive = False
319         mock_options.build_style = "debug"
320         mock_options.group = "api"
321         step = steps.RunTests(MockTool(log_executive=True), mock_options)
322         tool = MockTool(log_executive=True)
323         tool._deprecated_port = DeprecatedPort()
324         step = steps.RunTests(tool, mock_options)
325         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-api-tests', '--debug', '--json-output=/tmp/api_test_results.json'], cwd=/mock-checkout
326 """
327         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)