cfec29b333569036567a57cd0dfad4176413642d
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit_unittest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
3 # Copyright (C) 2010 Google Inc. All rights reserved.
4 #
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 # 1. Redistributions of source code must retain the above copyright
11 #    notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 #    notice, this list of conditions and the following disclaimer in the
14 #    documentation and/or other materials provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
17 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 import logging
29 import unittest
30
31 from webkitpy.common.system.executive_mock import MockExecutive
32 from webkitpy.common.system.filesystem_mock import MockFileSystem
33 from webkitpy.common.system.outputcapture import OutputCapture
34 from webkitpy.common.system.systemhost_mock import MockSystemHost
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.layout_tests.port import port_testcase
37 from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver
38 from webkitpy.layout_tests.port.config_mock import MockConfig
39 from webkitpy.tool.mocktool import MockOptions
40
41
42 class TestWebKitPort(WebKitPort):
43     port_name = "testwebkitport"
44
45     def __init__(self, symbols_string=None,
46                  expectations_file=None, skips_file=None, host=None, config=None,
47                  **kwargs):
48         self.symbols_string = symbols_string  # Passing "" disables all staticly-detectable features.
49         host = host or MockSystemHost()
50         config = config or MockConfig()
51         WebKitPort.__init__(self, host=host, config=config, **kwargs)
52
53     def all_test_configurations(self):
54         return [self.test_configuration()]
55
56     def _symbols_string(self):
57         return self.symbols_string
58
59     def _tests_for_other_platforms(self):
60         return ["media", ]
61
62     def _tests_for_disabled_features(self):
63         return ["accessibility", ]
64
65
66 class WebKitPortTest(port_testcase.PortTestCase):
67     port_name = 'webkit'
68     port_maker = TestWebKitPort
69
70     def test_check_build(self):
71         pass
72
73     def test_driver_cmd_line(self):
74         pass
75
76     def test_baseline_search_path(self):
77         pass
78
79     def test_path_to_test_expectations_file(self):
80         port = TestWebKitPort()
81         port._options = MockOptions(webkit_test_runner=False)
82         self.assertEqual(port.path_to_test_expectations_file(), '/mock-checkout/LayoutTests/platform/testwebkitport/TestExpectations')
83
84         port = TestWebKitPort()
85         port._options = MockOptions(webkit_test_runner=True)
86         self.assertEqual(port.path_to_test_expectations_file(), '/mock-checkout/LayoutTests/platform/testwebkitport/TestExpectations')
87
88         port = TestWebKitPort()
89         port.host.filesystem.files['/mock-checkout/LayoutTests/platform/testwebkitport/TestExpectations'] = 'some content'
90         port._options = MockOptions(webkit_test_runner=False)
91         self.assertEqual(port.path_to_test_expectations_file(), '/mock-checkout/LayoutTests/platform/testwebkitport/TestExpectations')
92
93     def test_skipped_directories_for_symbols(self):
94         # This first test confirms that the commonly found symbols result in the expected skipped directories.
95         symbols_string = " ".join(["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"])
96         expected_directories = set([
97             "mathml",  # Requires MathMLElement
98             "fast/canvas/webgl",  # Requires WebGLShader
99             "compositing/webgl",  # Requires WebGLShader
100             "http/tests/canvas/webgl",  # Requires WebGLShader
101             "mhtml",  # Requires MHTMLArchive
102             "fast/css/variables",  # Requires CSS Variables
103             "inspector/styles/variables",  # Requires CSS Variables
104         ])
105
106         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features(test_list=['mathml/foo.html']))
107         self.assertEqual(result_directories, expected_directories)
108
109         # Test that the nm string parsing actually works:
110         symbols_string = """
111 000000000124f498 s __ZZN7WebCore13GraphicsLayer12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__
112 000000000124f500 s __ZZN7WebCore13GraphicsLayer13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__
113 000000000124f670 s __ZZN7WebCore13GraphicsLayer13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__
114 """
115         # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked):
116         expected_directories = set(['mathml', 'transforms/3d', 'compositing/webgl', 'fast/canvas/webgl', 'animations/3d', 'mhtml', 'http/tests/canvas/webgl', 'fast/css/variables', 'inspector/styles/variables'])
117         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features(test_list=['mathml/foo.html']))
118         self.assertEqual(result_directories, expected_directories)
119
120     def test_skipped_directories_for_features(self):
121         supported_features = ["Accelerated Compositing", "Foo Feature"]
122         expected_directories = set(["animations/3d", "transforms/3d"])
123         port = TestWebKitPort(None, supported_features)
124         port._runtime_feature_list = lambda: supported_features
125         result_directories = set(port._skipped_tests_for_unsupported_features(test_list=["animations/3d/foo.html"]))
126         self.assertEqual(result_directories, expected_directories)
127
128     def test_skipped_directories_for_features_no_matching_tests_in_test_list(self):
129         supported_features = ["Accelerated Compositing", "Foo Feature"]
130         expected_directories = set([])
131         result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features(test_list=['foo.html']))
132         self.assertEqual(result_directories, expected_directories)
133
134     def test_skipped_tests_for_unsupported_features_empty_test_list(self):
135         supported_features = ["Accelerated Compositing", "Foo Feature"]
136         expected_directories = set([])
137         result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features(test_list=None))
138         self.assertEqual(result_directories, expected_directories)
139
140     def test_skipped_layout_tests(self):
141         self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(test_list=[]), set(['media']))
142
143     def test_skipped_file_search_paths(self):
144         port = TestWebKitPort()
145         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport']))
146         port._name = "testwebkitport-version"
147         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version']))
148         port._options = MockOptions(webkit_test_runner=True)
149         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version', 'testwebkitport-wk2', 'wk2']))
150         port._options = MockOptions(additional_platform_directory=["internal-testwebkitport"])
151         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version', 'internal-testwebkitport']))
152
153     def test_root_option(self):
154         port = TestWebKitPort()
155         port._options = MockOptions(root='/foo')
156         self.assertEqual(port._path_to_driver(), "/foo/DumpRenderTree")
157
158     def test_test_expectations(self):
159         # Check that we read the expectations file
160         host = MockSystemHost()
161         host.filesystem.write_text_file('/mock-checkout/LayoutTests/platform/testwebkitport/TestExpectations',
162             'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = TEXT\n')
163         port = TestWebKitPort(host=host)
164         self.assertEqual(''.join(port.expectations_dict().values()), 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = TEXT\n')
165
166     def test_build_driver(self):
167         output = OutputCapture()
168         port = TestWebKitPort()
169         # Delay setting _executive to avoid logging during construction
170         port._executive = MockExecutive(should_log=True)
171         port._options = MockOptions(configuration="Release")  # This should not be necessary, but I think TestWebKitPort is actually reading from disk (and thus detects the current configuration).
172         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
173         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
174
175         # Make sure when passed --webkit-test-runner we build the right tool.
176         port._options = MockOptions(webkit_test_runner=True, configuration="Release")
177         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\nMOCK run_command: ['Tools/Scripts/build-webkittestrunner', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
178         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
179
180         # Make sure we show the build log when --verbose is passed, which we simulate by setting the logging level to DEBUG.
181         output.set_log_level(logging.DEBUG)
182         port._options = MockOptions(configuration="Release")
183         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
184         expected_logs = "Output of ['Tools/Scripts/build-dumprendertree', '--release']:\nMOCK output of child process\n"
185         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
186         output.set_log_level(logging.INFO)
187
188         # Make sure that failure to build returns False.
189         port._executive = MockExecutive(should_log=True, should_throw=True)
190         # Because WK2 currently has to build both webkittestrunner and DRT, if DRT fails, that's the only one it tries.
191         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
192         expected_logs = "MOCK ScriptError\n\nMOCK output of child process\n"
193         self.assertFalse(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
194
195     def _assert_config_file_for_platform(self, port, platform, config_file):
196         self.assertEquals(port._apache_config_file_name_for_platform(platform), config_file)
197
198     def test_linux_distro_detection(self):
199         port = TestWebKitPort()
200         self.assertFalse(port._is_redhat_based())
201         self.assertFalse(port._is_debian_based())
202
203         port._filesystem = MockFileSystem({'/etc/redhat-release': ''})
204         self.assertTrue(port._is_redhat_based())
205         self.assertFalse(port._is_debian_based())
206
207         port._filesystem = MockFileSystem({'/etc/debian_version': ''})
208         self.assertFalse(port._is_redhat_based())
209         self.assertTrue(port._is_debian_based())
210
211     def test_apache_config_file_name_for_platform(self):
212         port = TestWebKitPort()
213         self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf')
214
215         self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf')
216         self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf')
217
218         port._is_redhat_based = lambda: True
219         self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd.conf')
220
221         port = TestWebKitPort()
222         port._is_debian_based = lambda: True
223         self._assert_config_file_for_platform(port, 'linux2', 'apache2-debian-httpd.conf')
224
225         self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf')
226         self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf')  # win32 isn't a supported sys.platform.  AppleWin/WinCairo/WinCE ports all use cygwin.
227         self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf')
228
229     def test_path_to_apache_config_file(self):
230         port = TestWebKitPort()
231         # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value.
232         port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf'
233         self.assertEquals(port._path_to_apache_config_file(), '/mock-checkout/LayoutTests/http/conf/httpd.conf')
234
235
236 class MockServerProcess(object):
237     def __init__(self, lines=None):
238         self.timed_out = False
239         self.lines = lines or []
240         self.crashed = False
241
242     def has_crashed(self):
243         return self.crashed
244
245     def read_stdout_line(self, deadline):
246         return self.lines.pop(0) + "\n"
247
248     def read_stdout(self, deadline, size):
249         # read_stdout doesn't actually function on lines, but this is sufficient for our testing.
250         line = self.lines.pop(0)
251         assert len(line) == size
252         return line
253
254     def read_either_stdout_or_stderr_line(self, deadline):
255         # FIXME: We should have tests which intermix stderr and stdout lines.
256         return self.read_stdout_line(deadline), None
257
258
259 class WebKitDriverTest(unittest.TestCase):
260     def test_read_block(self):
261         port = TestWebKitPort()
262         driver = WebKitDriver(port, 0, pixel_tests=False)
263         driver._server_process = MockServerProcess([
264             'ActualHash: foobar',
265             'Content-Type: my_type',
266             'Content-Transfer-Encoding: none',
267             "#EOF",
268         ])
269         content_block = driver._read_block(0)
270         self.assertEquals(content_block.content_type, 'my_type')
271         self.assertEquals(content_block.encoding, 'none')
272         self.assertEquals(content_block.content_hash, 'foobar')
273         driver._server_process = None
274
275     def test_read_binary_block(self):
276         port = TestWebKitPort()
277         driver = WebKitDriver(port, 0, pixel_tests=True)
278         driver._server_process = MockServerProcess([
279             'ActualHash: actual',
280             'ExpectedHash: expected',
281             'Content-Type: image/png',
282             'Content-Length: 8',
283             "12345678",
284             "#EOF",
285         ])
286         content_block = driver._read_block(0)
287         self.assertEquals(content_block.content_type, 'image/png')
288         self.assertEquals(content_block.content_hash, 'actual')
289         self.assertEquals(content_block.content, '12345678')
290         self.assertEquals(content_block.decoded_content, '12345678')
291         driver._server_process = None
292
293     def test_no_timeout(self):
294         port = TestWebKitPort()
295         driver = WebKitDriver(port, 0, pixel_tests=True, no_timeout=True)
296         self.assertEquals(driver.cmd_line(True, []), ['/mock-build/DumpRenderTree', '--no-timeout', '--pixel-tests', '-'])
297
298     def test_check_for_driver_crash(self):
299         port = TestWebKitPort()
300         driver = WebKitDriver(port, 0, pixel_tests=True)
301
302         class FakeServerProcess(object):
303             def __init__(self, crashed):
304                 self.crashed = crashed
305
306             def pid(self):
307                 return 1234
308
309             def name(self):
310                 return 'FakeServerProcess'
311
312             def has_crashed(self):
313                 return self.crashed
314
315             def stop(self):
316                 pass
317
318         def assert_crash(driver, error_line, crashed, name, pid, unresponsive=False):
319             self.assertEquals(driver._check_for_driver_crash(error_line), crashed)
320             self.assertEquals(driver._crashed_process_name, name)
321             self.assertEquals(driver._crashed_pid, pid)
322             self.assertEquals(driver._subprocess_was_unresponsive, unresponsive)
323             driver.stop()
324
325         driver._server_process = FakeServerProcess(False)
326         assert_crash(driver, '', False, None, None)
327
328         driver._crashed_process_name = None
329         driver._crashed_pid = None
330         driver._server_process = FakeServerProcess(False)
331         driver._subprocess_was_unresponsive = False
332         assert_crash(driver, '#CRASHED\n', True, 'FakeServerProcess', 1234)
333
334         driver._crashed_process_name = None
335         driver._crashed_pid = None
336         driver._server_process = FakeServerProcess(False)
337         driver._subprocess_was_unresponsive = False
338         assert_crash(driver, '#CRASHED - WebProcess\n', True, 'WebProcess', None)
339
340         driver._crashed_process_name = None
341         driver._crashed_pid = None
342         driver._server_process = FakeServerProcess(False)
343         driver._subprocess_was_unresponsive = False
344         assert_crash(driver, '#CRASHED - WebProcess (pid 8675)\n', True, 'WebProcess', 8675)
345
346         driver._crashed_process_name = None
347         driver._crashed_pid = None
348         driver._server_process = FakeServerProcess(False)
349         driver._subprocess_was_unresponsive = False
350         assert_crash(driver, '#PROCESS UNRESPONSIVE - WebProcess (pid 8675)\n', True, 'WebProcess', 8675, True)
351
352         driver._crashed_process_name = None
353         driver._crashed_pid = None
354         driver._server_process = FakeServerProcess(True)
355         driver._subprocess_was_unresponsive = False
356         assert_crash(driver, '', True, 'FakeServerProcess', 1234)
357
358     def test_creating_a_port_does_not_write_to_the_filesystem(self):
359         port = TestWebKitPort()
360         driver = WebKitDriver(port, 0, pixel_tests=True)
361         self.assertEquals(port._filesystem.written_files, {})
362         self.assertEquals(port._filesystem.last_tmpdir, None)
363
364     def test_stop_cleans_up_properly(self):
365         port = TestWebKitPort()
366         driver = WebKitDriver(port, 0, pixel_tests=True)
367         driver.start(True, [])
368         last_tmpdir = port._filesystem.last_tmpdir
369         self.assertNotEquals(last_tmpdir, None)
370         driver.stop()
371         self.assertFalse(port._filesystem.isdir(last_tmpdir))
372
373     def test_two_starts_cleans_up_properly(self):
374         port = TestWebKitPort()
375         driver = WebKitDriver(port, 0, pixel_tests=True)
376         driver.start(True, [])
377         last_tmpdir = port._filesystem.last_tmpdir
378         driver._start(True, [])
379         self.assertFalse(port._filesystem.isdir(last_tmpdir))