Make NRWT show DRT/WTR build logs when the build fails or --verbose is passed
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit_unittest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
3 # Copyright (C) 2010 Google Inc. All rights reserved.
4 #
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 # 1. Redistributions of source code must retain the above copyright
11 #    notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 #    notice, this list of conditions and the following disclaimer in the
14 #    documentation and/or other materials provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
17 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 import logging
29 import unittest
30
31 from webkitpy.common.host_mock import MockHost
32 from webkitpy.common.system.executive_mock import MockExecutive
33 from webkitpy.common.system.filesystem_mock import MockFileSystem
34 from webkitpy.common.system.outputcapture import OutputCapture
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.layout_tests.port import port_testcase
37 from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver
38 from webkitpy.tool.mocktool import MockOptions
39
40
41 class TestWebKitPort(WebKitPort):
42     port_name = "testwebkitport"
43
44     def __init__(self, symbols_string=None, feature_list=None,
45                  expectations_file=None, skips_file=None, host=None,
46                  **kwargs):
47         self.symbols_string = symbols_string  # Passing "" disables all staticly-detectable features.
48         self.feature_list = feature_list  # Passing [] disables all runtime-detectable features.
49         host = host or MockHost()
50         WebKitPort.__init__(self, host=host, **kwargs)
51
52     def all_test_configurations(self):
53         return [TestConfiguration.from_port(self)]
54
55     def _runtime_feature_list(self):
56         return self.feature_list
57
58     def _webcore_symbols_string(self):
59         return self.symbols_string
60
61     def _tests_for_other_platforms(self):
62         return ["media", ]
63
64     def _tests_for_disabled_features(self):
65         return ["accessibility", ]
66
67
68 class WebKitPortUnitTests(unittest.TestCase):
69     def test_default_options(self):
70         # The WebKit ports override new-run-webkit-test default options.
71         options = MockOptions(pixel_tests=None, time_out_ms=None)
72         port = WebKitPort(MockHost(), options=options)
73         self.assertEquals(port._options.pixel_tests, False)
74         self.assertEquals(port._options.time_out_ms, 35000)
75
76         # Note that we don't override options if specified by the user.
77         options = MockOptions(pixel_tests=True, time_out_ms=6000)
78         port = WebKitPort(MockHost(), options=options)
79         self.assertEquals(port._options.pixel_tests, True)
80         self.assertEquals(port._options.time_out_ms, 6000)
81
82
83 class WebKitPortTest(port_testcase.PortTestCase):
84     port_maker = TestWebKitPort
85
86     def test_check_build(self):
87         pass
88
89     def test_driver_cmd_line(self):
90         pass
91
92     def test_baseline_search_path(self):
93         pass
94
95     def test_skipped_directories_for_symbols(self):
96         # This first test confirms that the commonly found symbols result in the expected skipped directories.
97         symbols_string = " ".join(["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"])
98         expected_directories = set([
99             "mathml",  # Requires MathMLElement
100             "fast/canvas/webgl",  # Requires WebGLShader
101             "compositing/webgl",  # Requires WebGLShader
102             "http/tests/canvas/webgl",  # Requires WebGLShader
103             "mhtml",  # Requires MHTMLArchive
104         ])
105
106         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features())
107         self.assertEqual(result_directories, expected_directories)
108
109         # Test that the nm string parsing actually works:
110         symbols_string = """
111 000000000124f498 s __ZZN7WebCore13GraphicsLayer12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__
112 000000000124f500 s __ZZN7WebCore13GraphicsLayer13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__
113 000000000124f670 s __ZZN7WebCore13GraphicsLayer13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__
114 """
115         # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked):
116         expected_directories = set(['mathml', 'transforms/3d', 'compositing/webgl', 'fast/canvas/webgl', 'animations/3d', 'mhtml', 'http/tests/canvas/webgl'])
117         result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features())
118         self.assertEqual(result_directories, expected_directories)
119
120     def test_runtime_feature_list(self):
121         port = WebKitPort(MockHost())
122         port._executive.run_command = lambda command, cwd=None, error_handler=None: "Nonsense"
123         # runtime_features_list returns None when its results are meaningless (it couldn't run DRT or parse the output, etc.)
124         self.assertEquals(port._runtime_feature_list(), None)
125         port._executive.run_command = lambda command, cwd=None, error_handler=None: "SupportedFeatures:foo bar"
126         self.assertEquals(port._runtime_feature_list(), ['foo', 'bar'])
127
128     def test_skipped_directories_for_features(self):
129         supported_features = ["Accelerated Compositing", "Foo Feature"]
130         expected_directories = set(["animations/3d", "transforms/3d"])
131         result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features())
132         self.assertEqual(result_directories, expected_directories)
133
134     def test_skipped_layout_tests(self):
135         self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(), set(['media']))
136
137     def test_skipped_file_search_paths(self):
138         port = TestWebKitPort()
139         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport']))
140         port._name = "testwebkitport-version"
141         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version']))
142         port._options = MockOptions(webkit_test_runner=True)
143         self.assertEqual(port._skipped_file_search_paths(), set(['testwebkitport', 'testwebkitport-version', 'testwebkitport-wk2', 'wk2']))
144
145     def test_root_option(self):
146         port = TestWebKitPort()
147         port._options = MockOptions(root='/foo')
148         self.assertEqual(port._path_to_driver(), "/foo/DumpRenderTree")
149
150     def test_test_expectations(self):
151         # Check that we read both the expectations file and anything in a
152         # Skipped file, and that we include the feature and platform checks.
153         files = {
154             '/mock-checkout/LayoutTests/platform/testwebkitport/test_expectations.txt': 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n',
155             '/mock-checkout/LayoutTests/platform/testwebkitport/Skipped': 'fast/html/keygen.html',
156         }
157         mock_fs = MockFileSystem(files)
158         port = TestWebKitPort(filesystem=mock_fs)
159         self.assertEqual(port.test_expectations(),
160         """BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n
161 BUG_SKIPPED SKIP : fast/html/keygen.html = FAIL
162 BUG_SKIPPED SKIP : media = FAIL""")
163         files = {
164             '/mock-checkout/LayoutTests/platform/testwebkitport/test_expectations.txt': 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL',
165             '/mock-checkout/LayoutTests/platform/testwebkitport/Skipped': 'fast/html/keygen.html',
166         }
167         mock_fs = MockFileSystem(files)
168         port = TestWebKitPort(filesystem=mock_fs)
169         self.assertEqual(port.test_expectations(),
170         """BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL
171 BUG_SKIPPED SKIP : fast/html/keygen.html = FAIL
172 BUG_SKIPPED SKIP : media = FAIL""")
173
174
175     def test_build_driver(self):
176         output = OutputCapture()
177         port = TestWebKitPort()
178         # Delay setting _executive to avoid logging during construction
179         port._executive = MockExecutive(should_log=True)
180         port._options = MockOptions(configuration="Release")  # This should not be necessary, but I think TestWebKitPort is actually reading from disk (and thus detects the current configuration).
181         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
182         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
183
184         # Make sure when passed --webkit-test-runner we build the right tool.
185         port._options = MockOptions(webkit_test_runner=True, configuration="Release")
186         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\nMOCK run_command: ['Tools/Scripts/build-webkittestrunner', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
187         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=''))
188
189         # Make sure we show the build log when --verbose is passed, which we simulate by setting the logging level to DEBUG.
190         output.set_log_level(logging.DEBUG)
191         port._options = MockOptions(configuration="Release")
192         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
193         expected_logs = "Output of ['Tools/Scripts/build-dumprendertree', '--release']:\nMOCK output of child process\n"
194         self.assertTrue(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
195         output.set_log_level(logging.INFO)
196
197         # Make sure that failure to build returns False.
198         port._executive = MockExecutive(should_log=True, should_throw=True)
199         # Because WK2 currently has to build both webkittestrunner and DRT, if DRT fails, that's the only one it tries.
200         expected_stderr = "MOCK run_command: ['Tools/Scripts/build-dumprendertree', '--release'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'MOCK_ENVIRON_COPY': '1'}\n"
201         expected_logs = "MOCK ScriptError\n\nMOCK output of child process\n"
202         self.assertFalse(output.assert_outputs(self, port._build_driver, expected_stderr=expected_stderr, expected_logs=expected_logs))
203
204     def _assert_config_file_for_platform(self, port, platform, config_file):
205         self.assertEquals(port._apache_config_file_name_for_platform(platform), config_file)
206
207     def test_linux_distro_detection(self):
208         port = TestWebKitPort()
209         self.assertFalse(port._is_redhat_based())
210         self.assertFalse(port._is_debian_based())
211
212         port._filesystem = MockFileSystem({'/etc/redhat-release': ''})
213         self.assertTrue(port._is_redhat_based())
214         self.assertFalse(port._is_debian_based())
215
216         port._filesystem = MockFileSystem({'/etc/debian_version': ''})
217         self.assertFalse(port._is_redhat_based())
218         self.assertTrue(port._is_debian_based())
219
220     def test_apache_config_file_name_for_platform(self):
221         port = TestWebKitPort()
222         self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf')
223
224         self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf')
225         self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf')
226
227         port._is_redhat_based = lambda: True
228         self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd.conf')
229
230         port = TestWebKitPort()
231         port._is_debian_based = lambda: True
232         self._assert_config_file_for_platform(port, 'linux2', 'apache2-debian-httpd.conf')
233
234         self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf')
235         self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf')  # win32 isn't a supported sys.platform.  AppleWin/WinCairo/WinCE ports all use cygwin.
236         self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf')
237
238     def test_path_to_apache_config_file(self):
239         port = TestWebKitPort()
240         # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value.
241         port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf'
242         self.assertEquals(port._path_to_apache_config_file(), '/mock-checkout/LayoutTests/http/conf/httpd.conf')
243
244
245 class MockServerProcess(object):
246     def __init__(self, lines=None):
247         self.timed_out = False
248         self.crashed = False
249         self.lines = lines or []
250
251     def read_stdout_line(self, deadline):
252         return self.lines.pop(0) + "\n"
253
254     def read_stdout(self, deadline, size):
255         # read_stdout doesn't actually function on lines, but this is sufficient for our testing.
256         line = self.lines.pop(0)
257         assert len(line) == size
258         return line
259
260     def read_either_stdout_or_stderr_line(self, deadline):
261         # FIXME: We should have tests which intermix stderr and stdout lines.
262         return self.read_stdout_line(deadline), None
263
264
265 class WebKitDriverTest(unittest.TestCase):
266     def test_read_block(self):
267         port = TestWebKitPort()
268         driver = WebKitDriver(port, 0, pixel_tests=False)
269         driver._server_process = MockServerProcess([
270             'ActualHash: foobar',
271             'Content-Type: my_type',
272             'Content-Transfer-Encoding: none',
273             "#EOF",
274         ])
275         content_block = driver._read_block(0)
276         self.assertEquals(content_block.content_type, 'my_type')
277         self.assertEquals(content_block.encoding, 'none')
278         self.assertEquals(content_block.content_hash, 'foobar')
279
280     def test_read_binary_block(self):
281         port = TestWebKitPort()
282         driver = WebKitDriver(port, 0, pixel_tests=True)
283         driver._server_process = MockServerProcess([
284             'ActualHash: actual',
285             'ExpectedHash: expected',
286             'Content-Type: image/png',
287             'Content-Length: 8',
288             "12345678",
289             "#EOF",
290         ])
291         content_block = driver._read_block(0)
292         self.assertEquals(content_block.content_type, 'image/png')
293         self.assertEquals(content_block.content_hash, 'actual')
294         self.assertEquals(content_block.content, '12345678')
295         self.assertEquals(content_block.decoded_content, '12345678')