18d9b9df7178cde364f01217b505f9968dc038b2
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30 import optparse
31 import sys
32 import tempfile
33 import unittest
34
35 from webkitpy.common.system.executive import Executive, ScriptError
36 from webkitpy.common.system import executive_mock
37 from webkitpy.common.system.filesystem_mock import MockFileSystem
38 from webkitpy.common.system.outputcapture import OutputCapture
39 from webkitpy.common.system.path import abspath_to_uri
40 from webkitpy.thirdparty.mock import Mock
41 from webkitpy.tool.mocktool import MockOptions
42 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
43 from webkitpy.common.system.systemhost_mock import MockSystemHost
44
45 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
46 from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort
47
48 import config
49 import config_mock
50
51 class PortTest(unittest.TestCase):
52     def make_port(self, executive=None, with_tests=False, **kwargs):
53         host = MockSystemHost()
54         if executive:
55             host.executive = executive
56         if with_tests:
57             add_unit_tests_to_mock_filesystem(host.filesystem)
58             return TestPort(host, **kwargs)
59         return Port(host, **kwargs)
60
61     def test_default_child_processes(self):
62         port = self.make_port()
63         # Even though the MockPlatformInfo shows 1GB free memory (enough for 5 DRT instances)
64         # we're still limited by the 2 mock cores we have:
65         self.assertEqual(port.default_child_processes(), 2)
66         bytes_for_drt = 200 * 1024 * 1024
67
68         port.host.platform.free_bytes_memory = lambda: bytes_for_drt
69         expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
70         child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
71         self.assertEqual(child_processes, 1)
72
73         # Make sure that we always use one process, even if we don't have the memory for it.
74         port.host.platform.free_bytes_memory = lambda: bytes_for_drt - 1
75         expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
76         child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
77         self.assertEqual(child_processes, 1)
78
79     def test_format_wdiff_output_as_html(self):
80         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
81         html = self.make_port()._format_wdiff_output_as_html(output)
82         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
83         self.assertEqual(html, expected_html)
84
85     def test_wdiff_command(self):
86         port = self.make_port()
87         port._path_to_wdiff = lambda: "/path/to/wdiff"
88         command = port._wdiff_command("/actual/path", "/expected/path")
89         expected_command = [
90             "/path/to/wdiff",
91             "--start-delete=##WDIFF_DEL##",
92             "--end-delete=##WDIFF_END##",
93             "--start-insert=##WDIFF_ADD##",
94             "--end-insert=##WDIFF_END##",
95             "/actual/path",
96             "/expected/path",
97         ]
98         self.assertEqual(command, expected_command)
99
100     def _file_with_contents(self, contents, encoding="utf-8"):
101         new_file = tempfile.NamedTemporaryFile()
102         new_file.write(contents.encode(encoding))
103         new_file.flush()
104         return new_file
105
106     def test_pretty_patch_os_error(self):
107         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
108         oc = OutputCapture()
109         oc.capture_output()
110         self.assertEqual(port.pretty_patch_text("patch.txt"),
111                          port._pretty_patch_error_html)
112
113         # This tests repeated calls to make sure we cache the result.
114         self.assertEqual(port.pretty_patch_text("patch.txt"),
115                          port._pretty_patch_error_html)
116         oc.restore_output()
117
118     def test_pretty_patch_script_error(self):
119         # FIXME: This is some ugly white-box test hacking ...
120         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
121         port._pretty_patch_available = True
122         self.assertEqual(port.pretty_patch_text("patch.txt"),
123                          port._pretty_patch_error_html)
124
125         # This tests repeated calls to make sure we cache the result.
126         self.assertEqual(port.pretty_patch_text("patch.txt"),
127                          port._pretty_patch_error_html)
128
129     def integration_test_run_wdiff(self):
130         executive = Executive()
131         # This may fail on some systems.  We could ask the port
132         # object for the wdiff path, but since we don't know what
133         # port object to use, this is sufficient for now.
134         try:
135             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
136         except Exception, e:
137             wdiff_path = None
138
139         port = self.make_port(executive=executive)
140         port._path_to_wdiff = lambda: wdiff_path
141
142         if wdiff_path:
143             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
144             actual = self._file_with_contents(u"foo")
145             expected = self._file_with_contents(u"bar")
146             wdiff = port._run_wdiff(actual.name, expected.name)
147             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
148             self.assertEqual(wdiff, expected_wdiff)
149             # Running the full wdiff_text method should give the same result.
150             port._wdiff_available = True  # In case it's somehow already disabled.
151             wdiff = port.wdiff_text(actual.name, expected.name)
152             self.assertEqual(wdiff, expected_wdiff)
153             # wdiff should still be available after running wdiff_text with a valid diff.
154             self.assertTrue(port._wdiff_available)
155             actual.close()
156             expected.close()
157
158             # Bogus paths should raise a script error.
159             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
160             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
161             # wdiff will still be available after running wdiff_text with invalid paths.
162             self.assertTrue(port._wdiff_available)
163
164         # If wdiff does not exist _run_wdiff should throw an OSError.
165         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
166         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
167
168         # wdiff_text should not throw an error if wdiff does not exist.
169         self.assertEqual(port.wdiff_text("foo", "bar"), "")
170         # However wdiff should not be available after running wdiff_text if wdiff is missing.
171         self.assertFalse(port._wdiff_available)
172
173     def test_wdiff_text(self):
174         port = self.make_port()
175         port.wdiff_available = lambda: True
176         port._run_wdiff = lambda a, b: 'PASS'
177         self.assertEqual('PASS', port.wdiff_text(None, None))
178
179     def test_diff_text(self):
180         port = self.make_port()
181         # Make sure that we don't run into decoding exceptions when the
182         # filenames are unicode, with regular or malformed input (expected or
183         # actual input is always raw bytes, not unicode).
184         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
185         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
186         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
187
188         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
189         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
190
191         # Though expected and actual files should always be read in with no
192         # encoding (and be stored as str objects), test unicode inputs just to
193         # be safe.
194         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
195         port.diff_text(
196             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
197
198         # And make sure we actually get diff output.
199         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
200         self.assertTrue('foo' in diff)
201         self.assertTrue('bar' in diff)
202         self.assertTrue('exp.txt' in diff)
203         self.assertTrue('act.txt' in diff)
204         self.assertFalse('nosuchthing' in diff)
205
206     def test_default_configuration_notfound(self):
207         # Test that we delegate to the config object properly.
208         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
209         self.assertEqual(port.default_configuration(), 'default')
210
211     def test_layout_tests_skipping(self):
212         port = self.make_port()
213         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/media/video-zoom.html', '')
214         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/foo/bar.html', '')
215         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
216         self.assertTrue(port.skips_layout_test('foo/bar.html'))
217         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
218         self.assertFalse(port.skips_layout_test('foo/foo.html'))
219
220     def test_setup_test_run(self):
221         port = self.make_port()
222         # This routine is a no-op. We just test it for coverage.
223         port.setup_test_run()
224
225     def test_test_dirs(self):
226         port = self.make_port()
227         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '')
228         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '')
229         dirs = port.test_dirs()
230         self.assertTrue('canvas' in dirs)
231         self.assertTrue('css2.1' in dirs)
232
233     def test_get_option__set(self):
234         options, args = optparse.OptionParser().parse_args([])
235         options.foo = 'bar'
236         port = self.make_port(options=options)
237         self.assertEqual(port.get_option('foo'), 'bar')
238
239     def test_get_option__unset(self):
240         port = self.make_port()
241         self.assertEqual(port.get_option('foo'), None)
242
243     def test_get_option__default(self):
244         port = self.make_port()
245         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
246
247     def test_additional_platform_directory(self):
248         port = self.make_port(port_name='foo')
249         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
250         layout_test_dir = port.layout_tests_dir()
251         test_file = 'fast/test.html'
252
253         # No additional platform directory
254         self.assertEqual(
255             port.expected_baselines(test_file, '.txt'),
256             [(None, 'fast/test-expected.txt')])
257         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
258
259         # Simple additional platform directory
260         port._options.additional_platform_directory = ['/tmp/local-baselines']
261         port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo')
262         self.assertEqual(
263             port.expected_baselines(test_file, '.txt'),
264             [('/tmp/local-baselines', 'fast/test-expected.txt')])
265         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
266
267         # Multiple additional platform directories
268         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
269         self.assertEqual(
270             port.expected_baselines(test_file, '.txt'),
271             [('/tmp/local-baselines', 'fast/test-expected.txt')])
272         self.assertEqual(port.baseline_path(), '/foo')
273
274     def test_uses_test_expectations_file(self):
275         port = self.make_port(port_name='foo')
276         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
277         self.assertFalse(port.uses_test_expectations_file())
278         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
279         self.assertTrue(port.uses_test_expectations_file())
280
281     def test_find_no_paths_specified(self):
282         port = self.make_port(with_tests=True)
283         layout_tests_dir = port.layout_tests_dir()
284         tests = port.tests([])
285         self.assertNotEqual(len(tests), 0)
286
287     def test_find_one_test(self):
288         port = self.make_port(with_tests=True)
289         tests = port.tests(['failures/expected/image.html'])
290         self.assertEqual(len(tests), 1)
291
292     def test_find_glob(self):
293         port = self.make_port(with_tests=True)
294         tests = port.tests(['failures/expected/im*'])
295         self.assertEqual(len(tests), 2)
296
297     def test_find_with_skipped_directories(self):
298         port = self.make_port(with_tests=True)
299         tests = port.tests('userscripts')
300         self.assertTrue('userscripts/resources/iframe.html' not in tests)
301
302     def test_find_with_skipped_directories_2(self):
303         port = self.make_port(with_tests=True)
304         tests = port.tests(['userscripts/resources'])
305         self.assertEqual(tests, set([]))
306
307     def test_is_test_file(self):
308         filesystem = MockFileSystem()
309         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.html'))
310         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.shtml'))
311         self.assertTrue(Port._is_test_file(filesystem, '', 'test-ref-test.html'))
312         self.assertFalse(Port._is_test_file(filesystem, '', 'foo.png'))
313         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.html'))
314         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
315         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.html'))
316         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.html'))
317         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.xht'))
318         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.xhtml'))
319         self.assertFalse(Port._is_test_file(filesystem, '', 'ref-foo.html'))
320         self.assertFalse(Port._is_test_file(filesystem, '', 'notref-foo.xhr'))
321
322     def test_parse_reftest_list(self):
323         port = self.make_port(with_tests=True)
324         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
325         "",
326         "# some comment",
327         "!= test-2.html test-notref.html # more comments",
328         "== test-3.html test-ref.html",
329         "== test-3.html test-ref2.html",
330         "!= test-3.html test-notref.html"])
331
332         reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
333         self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
334             'bar/test-2.html': [('!=', 'bar/test-notref.html')],
335             'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
336
337     def test_operating_system(self):
338         self.assertEqual('mac', self.make_port().operating_system())
339
340     def test_check_httpd_success(self):
341         port = self.make_port(executive=MockExecutive2())
342         port._path_to_apache = lambda: '/usr/sbin/httpd'
343         capture = OutputCapture()
344         capture.capture_output()
345         self.assertTrue(port.check_httpd())
346         _, _, logs = capture.restore_output()
347         self.assertEqual('', logs)
348
349     def test_httpd_returns_error_code(self):
350         port = self.make_port(executive=MockExecutive2(exit_code=1))
351         port._path_to_apache = lambda: '/usr/sbin/httpd'
352         capture = OutputCapture()
353         capture.capture_output()
354         self.assertFalse(port.check_httpd())
355         _, _, logs = capture.restore_output()
356         self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
357
358     def assertVirtual(self, method, *args, **kwargs):
359         self.assertRaises(NotImplementedError, method, *args, **kwargs)
360
361     def test_virtual_methods(self):
362         port = Port(MockSystemHost())
363         self.assertVirtual(port.baseline_path)
364         self.assertVirtual(port.baseline_search_path)
365         self.assertVirtual(port.check_build, None)
366         self.assertVirtual(port.check_image_diff)
367         self.assertVirtual(port.create_driver, 0)
368         self.assertVirtual(port.diff_image, None, None)
369         self.assertVirtual(port.path_to_test_expectations_file)
370         self.assertVirtual(port.default_results_directory)
371         self.assertVirtual(port.test_expectations)
372         self.assertVirtual(port._path_to_apache)
373         self.assertVirtual(port._path_to_apache_config_file)
374         self.assertVirtual(port._path_to_driver)
375         self.assertVirtual(port._path_to_helper)
376         self.assertVirtual(port._path_to_image_diff)
377         self.assertVirtual(port._path_to_lighttpd)
378         self.assertVirtual(port._path_to_lighttpd_modules)
379         self.assertVirtual(port._path_to_lighttpd_php)
380         self.assertVirtual(port._path_to_wdiff)
381
382
383 if __name__ == '__main__':
384     unittest.main()