Use simctl instead of LayoutTestRelay
[WebKit.git] / Tools / Scripts / webkitpy / port / driver_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest
30
31 from webkitpy.common.system.systemhost_mock import MockSystemHost
32
33 from webkitpy.port import Port, Driver, DriverOutput
34 from webkitpy.port.server_process_mock import MockServerProcess
35 from webkitpy.thirdparty.mock import patch
36
37 # FIXME: remove the dependency on TestWebKitPort
38 from webkitpy.port.port_testcase import TestWebKitPort
39
40 from webkitpy.tool.mocktool import MockOptions
41
42 import os
43 import sys
44
45
46 class DriverOutputTest(unittest.TestCase):
47     def test_strip_metrics(self):
48         patterns = [
49             ('RenderView at (0,0) size 800x600', 'RenderView '),
50             ('text run at (0,0) width 100: "some text"', '"some text"'),
51             ('RenderBlock {HTML} at (0,0) size 800x600', 'RenderBlock {HTML} '),
52             ('RenderBlock {INPUT} at (29,3) size 12x12 [color=#000000]', 'RenderBlock {INPUT}'),
53
54             ('RenderBlock (floating) {DT} at (5,5) size 79x310 [border: (5px solid #000000)]',
55             'RenderBlock (floating) {DT} [border: px solid #000000)]'),
56
57             ('\n    "truncate text    "\n', '\n    "truncate text"\n'),
58
59             ('RenderText {#text} at (0,3) size 41x12\n    text run at (0,3) width 41: "whimper "\n',
60             'RenderText {#text} \n    "whimper"\n'),
61
62             ("""text run at (0,0) width 109: ".one {color: green;}"
63           text run at (109,0) width 0: " "
64           text run at (0,17) width 81: ".1 {color: red;}"
65           text run at (81,17) width 0: " "
66           text run at (0,34) width 102: ".a1 {color: green;}"
67           text run at (102,34) width 0: " "
68           text run at (0,51) width 120: "P.two {color: purple;}"
69           text run at (120,51) width 0: " "\n""",
70             '".one {color: green;}  .1 {color: red;}  .a1 {color: green;}  P.two {color: purple;}"\n'),
71
72             ('text-- other text', 'text--other text'),
73
74             (' some output   "truncate trailing spaces at end of line after text"   \n',
75             ' some output   "truncate trailing spaces at end of line after text"\n'),
76
77             (r'scrollWidth 120', r'scrollWidth'),
78             (r'scrollHeight 120', r'scrollHeight'),
79         ]
80
81         for pattern in patterns:
82             driver_output = DriverOutput(pattern[0], None, None, None)
83             driver_output.strip_metrics()
84             self.assertEqual(driver_output.text, pattern[1])
85
86
87 class DriverTest(unittest.TestCase):
88     def make_port(self, host=None, options=None):
89         port = Port(host or MockSystemHost(), 'test', options or MockOptions(configuration='Release'))
90         port._config.build_directory = lambda configuration: '/mock-build'
91         return port
92
93     def _assert_wrapper(self, wrapper_string, expected_wrapper):
94         wrapper = Driver(self.make_port(options=MockOptions(wrapper=wrapper_string)), None, pixel_tests=False)._command_wrapper()
95         self.assertEqual(wrapper, expected_wrapper)
96
97     def test_command_wrapper(self):
98         self._assert_wrapper(None, [])
99         self._assert_wrapper("valgrind", ["valgrind"])
100
101         # Validate that shlex works as expected.
102         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
103         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
104         self._assert_wrapper(command_with_spaces, expected_parse)
105
106     def test_profiler_as_wrapper(self):
107         driver = Driver(self.make_port(MockSystemHost(os_name='linux'), MockOptions(profile=True, profiler='perf')), None, False)
108         self.assertEqual(driver._command_wrapper(), ['perf', 'record', '-g', '--output', '/mock-build/layout-test-results/test.data'])
109
110     def test_profiler_and_wrapper(self):
111         driver = Driver(self.make_port(MockSystemHost(os_name='linux'), MockOptions(profile=True, profiler='perf', wrapper='valgrind')), None, False)
112         self.assertEqual(driver._command_wrapper(), ['valgrind', 'perf', 'record', '-g', '--output', '/mock-build/layout-test-results/test.data'])
113
114     def test_test_to_uri(self):
115         port = self.make_port()
116         driver = Driver(port, None, pixel_tests=False)
117         self.assertEqual(driver.test_to_uri('foo/bar.html'), 'file://%s/foo/bar.html' % port.layout_tests_dir())
118         self.assertEqual(driver.test_to_uri('http/tests/foo.html'), 'http://127.0.0.1:8000/foo.html')
119         self.assertEqual(driver.test_to_uri('http/tests/ssl/bar.html'), 'https://127.0.0.1:8443/ssl/bar.html')
120         self.assertEqual(driver.test_to_uri('imported/w3c/web-platform-tests/foo/bar.html'), 'http://localhost:8800/foo/bar.html')
121
122     def test_uri_to_test(self):
123         port = self.make_port()
124         driver = Driver(port, None, pixel_tests=False)
125         self.assertEqual(driver.uri_to_test('file://%s/foo/bar.html' % port.layout_tests_dir()), 'foo/bar.html')
126         self.assertEqual(driver.uri_to_test('http://127.0.0.1:8000/foo.html'), 'http/tests/foo.html')
127         self.assertEqual(driver.uri_to_test('https://127.0.0.1:8443/ssl/bar.html'), 'http/tests/ssl/bar.html')
128         self.assertEqual(driver.uri_to_test('http://localhost:8800/foo/bar.html'), 'imported/w3c/web-platform-tests/foo/bar.html')
129
130     def test_read_block(self):
131         port = TestWebKitPort()
132         driver = Driver(port, 0, pixel_tests=False)
133         driver._server_process = MockServerProcess(lines=[
134             'ActualHash: foobar',
135             'Content-Type: my_type',
136             'Content-Transfer-Encoding: none',
137             "#EOF",
138         ])
139         content_block = driver._read_block(0, "")
140         self.assertEqual(content_block.content_type, 'my_type')
141         self.assertEqual(content_block.encoding, 'none')
142         self.assertEqual(content_block.content_hash, 'foobar')
143         driver._server_process = None
144
145     def test_read_binary_block(self):
146         port = TestWebKitPort()
147         driver = Driver(port, 0, pixel_tests=True)
148         driver._server_process = MockServerProcess(lines=[
149             'ActualHash: actual',
150             'ExpectedHash: expected',
151             'Content-Type: image/png',
152             'Content-Length: 9',
153             "12345678",
154             "#EOF",
155         ])
156         content_block = driver._read_block(0, "")
157         self.assertEqual(content_block.content_type, 'image/png')
158         self.assertEqual(content_block.content_hash, 'actual')
159         self.assertEqual(content_block.content, '12345678\n')
160         self.assertEqual(content_block.decoded_content, '12345678\n')
161         driver._server_process = None
162
163     def test_read_base64_block(self):
164         port = TestWebKitPort()
165         driver = Driver(port, 0, pixel_tests=True)
166         driver._server_process = MockServerProcess(lines=[
167             'ActualHash: actual',
168             'ExpectedHash: expected',
169             'Content-Type: image/png',
170             'Content-Transfer-Encoding: base64',
171             'Content-Length: 12',
172             'MTIzNDU2NzgK#EOF',
173         ])
174         content_block = driver._read_block(0, "")
175         self.assertEqual(content_block.content_type, 'image/png')
176         self.assertEqual(content_block.content_hash, 'actual')
177         self.assertEqual(content_block.encoding, 'base64')
178         self.assertEqual(content_block.content, 'MTIzNDU2NzgK')
179         self.assertEqual(content_block.decoded_content, '12345678\n')
180
181     def test_no_timeout(self):
182         port = TestWebKitPort()
183         port._config.build_directory = lambda configuration: '/mock-build'
184         driver = Driver(port, 0, pixel_tests=True, no_timeout=True)
185         if sys.platform.startswith('win'):
186             self.assertEqual(driver.cmd_line(True, []), ['/mock-build/DumpRenderTree.exe', '--no-timeout', '-'])
187         else:
188             self.assertEqual(driver.cmd_line(True, []), ['/mock-build/DumpRenderTree', '--no-timeout', '-'])
189
190     def test_check_for_driver_crash(self):
191         port = TestWebKitPort()
192         driver = Driver(port, 0, pixel_tests=True)
193
194         class FakeServerProcess(object):
195             def __init__(self, crashed):
196                 self.crashed = crashed
197
198             def pid(self):
199                 return 1234
200
201             def name(self):
202                 return 'FakeServerProcess'
203
204             def has_crashed(self):
205                 return self.crashed
206
207             def stop(self, timeout):
208                 pass
209
210             def write(self, bytes, ignore_crash=False):
211                 pass
212
213         def assert_crash(driver, error_line, crashed, name, pid, unresponsive=False):
214             self.assertEqual(driver._check_for_driver_crash_or_unresponsiveness(error_line), crashed)
215             self.assertEqual(driver._crashed_process_name, name)
216             self.assertEqual(driver._crashed_pid, pid)
217             self.assertEqual(driver._driver_timed_out, unresponsive)
218             driver.stop()
219
220         driver._server_process = FakeServerProcess(False)
221         assert_crash(driver, '', False, None, None)
222
223         driver._crashed_process_name = None
224         driver._crashed_pid = None
225         driver._server_process = FakeServerProcess(False)
226         driver._driver_timed_out = False
227         assert_crash(driver, '#CRASHED\n', True, 'FakeServerProcess', 1234)
228
229         driver._crashed_process_name = None
230         driver._crashed_pid = None
231         driver._server_process = FakeServerProcess(False)
232         driver._driver_timed_out = False
233         assert_crash(driver, '#CRASHED - WebProcess\n', True, 'WebProcess', None)
234
235         driver._crashed_process_name = None
236         driver._crashed_pid = None
237         driver._server_process = FakeServerProcess(False)
238         driver._driver_timed_out = False
239         assert_crash(driver, '#CRASHED - WebProcess (pid 8675)\n', True, 'WebProcess', 8675)
240
241         driver._crashed_process_name = None
242         driver._crashed_pid = None
243         driver._server_process = FakeServerProcess(False)
244         driver._driver_timed_out = False
245         assert_crash(driver, '#PROCESS UNRESPONSIVE - WebProcess (pid 8675)\n', True, None, None, True)
246
247         driver._crashed_process_name = None
248         driver._crashed_pid = None
249         driver._server_process = FakeServerProcess(False)
250         driver._driver_timed_out = False
251         assert_crash(driver, '#CRASHED - renderer (pid 8675)\n', True, 'renderer', 8675)
252
253         driver._crashed_process_name = None
254         driver._crashed_pid = None
255         driver._server_process = FakeServerProcess(True)
256         driver._driver_timed_out = False
257         assert_crash(driver, '', True, 'FakeServerProcess', 1234)
258
259     def test_creating_a_port_does_not_write_to_the_filesystem(self):
260         port = TestWebKitPort()
261         driver = Driver(port, 0, pixel_tests=True)
262         self.assertEqual(port._filesystem.written_files, {})
263         self.assertEqual(port._filesystem.last_tmpdir, None)
264
265     def test_stop_cleans_up_properly(self):
266         port = TestWebKitPort()
267         port._test_runner_process_constructor = MockServerProcess
268         driver = Driver(port, 0, pixel_tests=True)
269         driver.start(True, [])
270         last_tmpdir = port._filesystem.last_tmpdir
271         self.assertNotEquals(last_tmpdir, None)
272         driver.stop()
273         self.assertFalse(port._filesystem.isdir(last_tmpdir))
274
275     def test_two_starts_cleans_up_properly(self):
276         port = TestWebKitPort()
277         port._test_runner_process_constructor = MockServerProcess
278         driver = Driver(port, 0, pixel_tests=True)
279         driver.start(True, [])
280         last_tmpdir = port._filesystem.last_tmpdir
281         driver._start(True, [])
282         self.assertFalse(port._filesystem.isdir(last_tmpdir))
283
284     def test_start_actually_starts(self):
285         port = TestWebKitPort()
286         port._test_runner_process_constructor = MockServerProcess
287         driver = Driver(port, 0, pixel_tests=True)
288         driver.start(True, [])
289         self.assertTrue(driver._server_process.started)
290
291     def test__append_environment_variable_path(self):
292         port = self.make_port()
293         driver = Driver(port, None, pixel_tests=False)
294
295         environment = {}
296         variable = 'DYLD_FAKE_PATH'
297         path = '/tmp/fake-dyld-path'
298         driver._append_environment_variable_path(environment, variable, path)
299         self.assertEqual(environment[variable], path)
300
301         base_value = ''
302         environment[variable] = base_value
303         driver._append_environment_variable_path(environment, variable, path)
304         self.assertEqual(environment[variable], base_value + os.pathsep + path)
305
306         base_value = '/usr/lib'
307         environment[variable] = base_value
308         driver._append_environment_variable_path(environment, variable, path)
309         self.assertEqual(environment[variable], base_value + os.pathsep + path)
310
311         base_value = os.pathsep.join(['/usr/lib', '/usr/local/lib'])
312         environment[variable] = base_value
313         driver._append_environment_variable_path(environment, variable, path)
314         self.assertEqual(environment[variable], base_value + os.pathsep + path)
315
316     def test_setup_environ_for_test(self):
317         environment_user = {}
318         environment_user['WEBKIT_OUTPUTDIR'] = '/opt/webkit/WebKitBuild/Release'
319         environment_user['FOO'] = 'BAR'
320         with patch('os.environ', environment_user):
321             port = self.make_port()
322             driver = Driver(port, None, pixel_tests=False)
323             environment_driver_test = driver._setup_environ_for_test()
324             self.assertNotIn('FOO', environment_driver_test)
325             self.assertIn('WEBKIT_OUTPUTDIR', environment_driver_test)
326             self.assertEqual(environment_user['WEBKIT_OUTPUTDIR'], environment_driver_test['WEBKIT_OUTPUTDIR'])