Some perf tests time out when ran by run-perf-tests
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit_unittest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
3 # Copyright (C) 2010 Google Inc. All rights reserved.
4 #
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 # 1. Redistributions of source code must retain the above copyright
11 #    notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 #    notice, this list of conditions and the following disclaimer in the
14 #    documentation and/or other materials provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
17 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 import logging
29 import unittest
30
31 from webkitpy.common.system.executive_mock import MockExecutive
32 from webkitpy.common.system.filesystem_mock import MockFileSystem
33 from webkitpy.common.system.outputcapture import OutputCapture
34 from webkitpy.common.system.systemhost_mock import MockSystemHost
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.layout_tests.port import port_testcase
37 from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver
38 from webkitpy.tool.mocktool import MockOptions
39
40
41 class TestWebKitPort(WebKitPort):
42     port_name = "testwebkitport"
43
44     def __init__(self, symbols_string=None, feature_list=None,
45                  expectations_file=None, skips_file=None, host=None,
46                  **kwargs):
47         self.symbols_string = symbols_string  # Passing "" disables all staticly-detectable features.
48         self.feature_list = feature_list  # Passing [] disables all runtime-detectable features.
49         host = host or MockSystemHost()
50         WebKitPort.__init__(self, host=host, **kwargs)
51
52     def all_test_configurations(self):
53         return [self.test_configuration()]
54
55     def _runtime_feature_list(self):
56         return self.feature_list
57
58     def _webcore_symbols_string(self):
59         return self.symbols_string
60
61     def _tests_for_other_platforms(self):
62         return ["media", ]
63
64     def _tests_for_disabled_features(self):
65         return ["accessibility", ]
66
67
68 class WebKitPortUnitTests(unittest.TestCase):
69     def test_default_options(self):
70         # The WebKit ports override new-run-webkit-test default options.
71         options = MockOptions(pixel_tests=None, time_out_ms=None)
72         port = WebKitPort(MockSystemHost(), options=options)
73         self.assertEquals(port._options.pixel_tests, False)
74         self.assertEquals(port._options.time_out_ms, 35000)
75
76         # Note that we don't override options if specified by the user.
77         options = MockOptions(pixel_tests=True, time_out_ms=6000)
78         port = WebKitPort(MockSystemHost(), options=options)
79         self.assertEquals(port._options.pixel_tests, True)
80         self.assertEquals(port._options.time_out_ms, 6000)
81
82
83 class WebKitPortTest(port_testcase.PortTestCase):
84     port_name = 'webkit'
85     port_maker = TestWebKitPort
86
87     def test_check_build(self):
88         pass
89
90     def test_driver_cmd_line(self):
91         pass
92
93     def test_baseline_search_path(self):
94         pass
95
96     def test_path_to_test_expectations_file(self):
97         port = TestWebKitPort()
98         port._options = MockOptions(webkit_test_runner=False)
99         self.assertEqual(port.path_to_test_expectations_file(), '/mock-checkout/LayoutTests/platform/testwebkitport/test_expectations.txt')
100         port._options = MockOptions(webkit_test_runner=True)
101         self.assertEqual(port.path_to_test_expectations_file(), '/mock-checkout/LayoutTests/platform/testwebkitport/test_expectations.txt')
102
103     def test_skipped_directories_for_symbols(self):
104         # This first test confirms that the commonly found symbols result in the expected skipped directories.
105         symbols_string = " ".join(["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"])
106         expected_directories = set([
107             "mathml",  # Requires MathMLElement
108             "fast/canvas/webgl",  # Requires WebGLShader
109             "compositing/webgl",  # Requires WebGLShader
110             "http/tests/canvas/webgl",  # Requires WebGLShader
111             "mhtml",  # Requires MHTMLArchive
112         ])
113
114         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features())
115         self.assertEqual(result_directories, expected_directories)
116
117         # Test that the nm string parsing actually works:
118         symbols_string = """
119 000000000124f498 s __ZZN7WebCore13GraphicsLayer12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__
120 000000000124f500 s __ZZN7WebCore13GraphicsLayer13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__
121 000000000124f670 s __ZZN7WebCore13GraphicsLayer13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__
122 """
123         # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked):
124         expected_directories = set(['mathml', 'transforms/3d', 'compositing/webgl', 'fast/canvas/webgl', 'animations/3d', 'mhtml', 'http/tests/canvas/webgl'])
125         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features())
126         self.assertEqual(result_directories, expected_directories)
127
128     def test_runtime_feature_list(self):
129         port = WebKitPort(MockSystemHost())
130         port._executive.run_command = lambda command, cwd=None, error_handler=None: "Nonsense"
131         # runtime_features_list returns None when its results are meaningless (it couldn't run DRT or parse the output, etc.)
132         self.assertEquals(port._runtime_feature_list(), None)
133         port._executive.run_command = lambda command, cwd=None, error_handler=None: "SupportedFeatures:foo bar"
134         self.assertEquals(port._runtime_feature_list(), ['foo', 'bar'])
135
136     def test_skipped_directories_for_features(self):
137         supported_features = ["Accelerated Compositing", "Foo Feature"]
138         expected_directories = set(["animations/3d", "transforms/3d"])
139         result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features())
140         self.assertEqual(result_directories, expected_directories)
141
142     def test_skipped_layout_tests(self):
143         self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(), set(['media']))
144
145     def test_skipped_file_search_paths(self):
146         port = TestWebKitPort()
147         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport']))
148         port._name = "testwebkitport-version"
149         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version']))
150         port._options = MockOptions(webkit_test_runner=True)
151         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version', 'testwebkitport-wk2', 'wk2']))
152
153     def test_root_option(self):
154         port = TestWebKitPort()
155         port._options = MockOptions(root='/foo')
156         self.assertEqual(port._path_to_driver(), "/foo/DumpRenderTree")
157
158     def test_test_expectations(self):
159         # Check that we read the expectations file
160         host = MockSystemHost()
161         host.filesystem.write_text_file('/mock-checkout/LayoutTests/platform/testwebkitport/test_expectations.txt',
162             'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n')
163         port = TestWebKitPort(host=host)
164         self.assertEqual(port.test_expectations(), 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n')
165
166     def test_build_driver(self):
167         output = OutputCapture()
168         port = TestWebKitPort()
169         # Delay setting _executive to avoid logging during construction
170         port._executive = MockExecutive(should_log=True)
171         port._options = MockOptions(configuration="Release")  # This should not be necessary, but I think TestWebKitPort is actually reading from disk (and thus detects the current configuration).
172         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
173         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
174
175         # Make sure when passed --webkit-test-runner we build the right tool.
176         port._options = MockOptions(webkit_test_runner=True, configuration="Release")
177         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\nMOCK run_command: ['Tools/Scripts/build-webkittestrunner', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
178         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
179
180         # Make sure we show the build log when --verbose is passed, which we simulate by setting the logging level to DEBUG.
181         output.set_log_level(logging.DEBUG)
182         port._options = MockOptions(configuration="Release")
183         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
184         expected_logs = "Output of ['Tools/Scripts/build-dumprendertree', '--release']:\nMOCK output of child process\n"
185         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
186         output.set_log_level(logging.INFO)
187
188         # Make sure that failure to build returns False.
189         port._executive = MockExecutive(should_log=True, should_throw=True)
190         # Because WK2 currently has to build both webkittestrunner and DRT, if DRT fails, that's the only one it tries.
191         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
192         expected_logs = "MOCK ScriptError\n\nMOCK output of child process\n"
193         self.assertFalse(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
194
195     def _assert_config_file_for_platform(self, port, platform, config_file):
196         self.assertEquals(port._apache_config_file_name_for_platform(platform), config_file)
197
198     def test_linux_distro_detection(self):
199         port = TestWebKitPort()
200         self.assertFalse(port._is_redhat_based())
201         self.assertFalse(port._is_debian_based())
202
203         port._filesystem = MockFileSystem({'/etc/redhat-release': ''})
204         self.assertTrue(port._is_redhat_based())
205         self.assertFalse(port._is_debian_based())
206
207         port._filesystem = MockFileSystem({'/etc/debian_version': ''})
208         self.assertFalse(port._is_redhat_based())
209         self.assertTrue(port._is_debian_based())
210
211     def test_apache_config_file_name_for_platform(self):
212         port = TestWebKitPort()
213         self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf')
214
215         self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf')
216         self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf')
217
218         port._is_redhat_based = lambda: True
219         self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd.conf')
220
221         port = TestWebKitPort()
222         port._is_debian_based = lambda: True
223         self._assert_config_file_for_platform(port, 'linux2', 'apache2-debian-httpd.conf')
224
225         self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf')
226         self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf')  # win32 isn't a supported sys.platform.  AppleWin/WinCairo/WinCE ports all use cygwin.
227         self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf')
228
229     def test_path_to_apache_config_file(self):
230         port = TestWebKitPort()
231         # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value.
232         port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf'
233         self.assertEquals(port._path_to_apache_config_file(), '/mock-checkout/LayoutTests/http/conf/httpd.conf')
234
235
236 class MockServerProcess(object):
237     def __init__(self, lines=None):
238         self.timed_out = False
239         self.crashed = False
240         self.lines = lines or []
241
242     def read_stdout_line(self, deadline):
243         return self.lines.pop(0) + "\n"
244
245     def read_stdout(self, deadline, size):
246         # read_stdout doesn't actually function on lines, but this is sufficient for our testing.
247         line = self.lines.pop(0)
248         assert len(line) == size
249         return line
250
251     def read_either_stdout_or_stderr_line(self, deadline):
252         # FIXME: We should have tests which intermix stderr and stdout lines.
253         return self.read_stdout_line(deadline), None
254
255
256 class WebKitDriverTest(unittest.TestCase):
257     def test_read_block(self):
258         port = TestWebKitPort()
259         driver = WebKitDriver(port, 0, pixel_tests=False)
260         driver._server_process = MockServerProcess([
261             'ActualHash: foobar',
262             'Content-Type: my_type',
263             'Content-Transfer-Encoding: none',
264             "#EOF",
265         ])
266         content_block = driver._read_block(0)
267         self.assertEquals(content_block.content_type, 'my_type')
268         self.assertEquals(content_block.encoding, 'none')
269         self.assertEquals(content_block.content_hash, 'foobar')
270
271     def test_read_binary_block(self):
272         port = TestWebKitPort()
273         driver = WebKitDriver(port, 0, pixel_tests=True)
274         driver._server_process = MockServerProcess([
275             'ActualHash: actual',
276             'ExpectedHash: expected',
277             'Content-Type: image/png',
278             'Content-Length: 8',
279             "12345678",
280             "#EOF",
281         ])
282         content_block = driver._read_block(0)
283         self.assertEquals(content_block.content_type, 'image/png')
284         self.assertEquals(content_block.content_hash, 'actual')
285         self.assertEquals(content_block.content, '12345678')
286         self.assertEquals(content_block.decoded_content, '12345678')
287
288     def test_no_timeout(self):
289         port = TestWebKitPort()
290         driver = WebKitDriver(port, 0, pixel_tests=True, no_timeout=True)
291         self.assertEquals(driver.cmd_line(), ['MOCK output of child process/DumpRenderTree', '--pixel-tests', '--no-timeout', '-'])