d705afc4b6be21afadff525a496e505299e5962b
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import optparse
30 import sys
31 import tempfile
32 import unittest
33
34 from webkitpy.common.system.executive import Executive, ScriptError
35 from webkitpy.common.system import executive_mock
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system import outputcapture
38 from webkitpy.common.system.path import abspath_to_uri
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.mocktool import MockOptions
41 from webkitpy.common.system.executive_mock import MockExecutive
42 from webkitpy.common.host_mock import MockHost
43
44 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
45
46 import config
47 import config_mock
48
49
50 class PortTest(unittest.TestCase):
51     def make_port(self, host=None, **kwargs):
52         host = host or MockHost()
53         return Port(host, **kwargs)
54
55     def test_format_wdiff_output_as_html(self):
56         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
57         html = self.make_port()._format_wdiff_output_as_html(output)
58         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
59         self.assertEqual(html, expected_html)
60
61     def test_wdiff_command(self):
62         port = self.make_port()
63         port._path_to_wdiff = lambda: "/path/to/wdiff"
64         command = port._wdiff_command("/actual/path", "/expected/path")
65         expected_command = [
66             "/path/to/wdiff",
67             "--start-delete=##WDIFF_DEL##",
68             "--end-delete=##WDIFF_END##",
69             "--start-insert=##WDIFF_ADD##",
70             "--end-insert=##WDIFF_END##",
71             "/actual/path",
72             "/expected/path",
73         ]
74         self.assertEqual(command, expected_command)
75
76     def _file_with_contents(self, contents, encoding="utf-8"):
77         new_file = tempfile.NamedTemporaryFile()
78         new_file.write(contents.encode(encoding))
79         new_file.flush()
80         return new_file
81
82     def test_pretty_patch_os_error(self):
83         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
84         oc = outputcapture.OutputCapture()
85         oc.capture_output()
86         self.assertEqual(port.pretty_patch_text("patch.txt"),
87                          port._pretty_patch_error_html)
88
89         # This tests repeated calls to make sure we cache the result.
90         self.assertEqual(port.pretty_patch_text("patch.txt"),
91                          port._pretty_patch_error_html)
92         oc.restore_output()
93
94     def test_pretty_patch_script_error(self):
95         # FIXME: This is some ugly white-box test hacking ...
96         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
97         port._pretty_patch_available = True
98         self.assertEqual(port.pretty_patch_text("patch.txt"),
99                          port._pretty_patch_error_html)
100
101         # This tests repeated calls to make sure we cache the result.
102         self.assertEqual(port.pretty_patch_text("patch.txt"),
103                          port._pretty_patch_error_html)
104
105     def integration_test_run_wdiff(self):
106         executive = Executive()
107         # This may fail on some systems.  We could ask the port
108         # object for the wdiff path, but since we don't know what
109         # port object to use, this is sufficient for now.
110         try:
111             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
112         except Exception, e:
113             wdiff_path = None
114
115         port = self.make_port(executive=executive)
116         port._path_to_wdiff = lambda: wdiff_path
117
118         if wdiff_path:
119             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
120             actual = self._file_with_contents(u"foo")
121             expected = self._file_with_contents(u"bar")
122             wdiff = port._run_wdiff(actual.name, expected.name)
123             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
124             self.assertEqual(wdiff, expected_wdiff)
125             # Running the full wdiff_text method should give the same result.
126             port._wdiff_available = True  # In case it's somehow already disabled.
127             wdiff = port.wdiff_text(actual.name, expected.name)
128             self.assertEqual(wdiff, expected_wdiff)
129             # wdiff should still be available after running wdiff_text with a valid diff.
130             self.assertTrue(port._wdiff_available)
131             actual.close()
132             expected.close()
133
134             # Bogus paths should raise a script error.
135             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
136             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
137             # wdiff will still be available after running wdiff_text with invalid paths.
138             self.assertTrue(port._wdiff_available)
139
140         # If wdiff does not exist _run_wdiff should throw an OSError.
141         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
142         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
143
144         # wdiff_text should not throw an error if wdiff does not exist.
145         self.assertEqual(port.wdiff_text("foo", "bar"), "")
146         # However wdiff should not be available after running wdiff_text if wdiff is missing.
147         self.assertFalse(port._wdiff_available)
148
149     def test_diff_text(self):
150         port = self.make_port()
151         # Make sure that we don't run into decoding exceptions when the
152         # filenames are unicode, with regular or malformed input (expected or
153         # actual input is always raw bytes, not unicode).
154         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
155         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
156         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
157
158         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
159         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
160
161         # Though expected and actual files should always be read in with no
162         # encoding (and be stored as str objects), test unicode inputs just to
163         # be safe.
164         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
165         port.diff_text(
166             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
167
168         # And make sure we actually get diff output.
169         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
170         self.assertTrue('foo' in diff)
171         self.assertTrue('bar' in diff)
172         self.assertTrue('exp.txt' in diff)
173         self.assertTrue('act.txt' in diff)
174         self.assertFalse('nosuchthing' in diff)
175
176     def test_default_configuration_notfound(self):
177         # Test that we delegate to the config object properly.
178         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
179         self.assertEqual(port.default_configuration(), 'default')
180
181     def test_layout_tests_skipping(self):
182         filesystem = MockFileSystem({
183             '/mock-checkout/LayoutTests/media/video-zoom.html': '',
184             '/mock-checkout/LayoutTests/foo/bar.html': '',
185         })
186         port = self.make_port(filesystem=filesystem)
187         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
188         self.assertTrue(port.skips_layout_test('foo/bar.html'))
189         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
190         self.assertFalse(port.skips_layout_test('foo/foo.html'))
191
192     def test_setup_test_run(self):
193         port = self.make_port()
194         # This routine is a no-op. We just test it for coverage.
195         port.setup_test_run()
196
197     def test_test_dirs(self):
198         filesystem = MockFileSystem({
199             '/mock-checkout/LayoutTests/canvas/test': '',
200             '/mock-checkout/LayoutTests/css2.1/test': '',
201         })
202         port = self.make_port(filesystem=filesystem)
203         dirs = port.test_dirs()
204         self.assertTrue('canvas' in dirs)
205         self.assertTrue('css2.1' in dirs)
206
207     def test_test_to_uri(self):
208         port = self.make_port()
209         layout_test_dir = port.layout_tests_dir()
210         test = 'foo/bar.html'
211         path = port._filesystem.join(layout_test_dir, test)
212         if sys.platform == 'win32':
213             path = path.replace("\\", "/")
214
215         self.assertEqual(port.test_to_uri(test), abspath_to_uri(path))
216
217     def test_get_option__set(self):
218         options, args = optparse.OptionParser().parse_args([])
219         options.foo = 'bar'
220         port = self.make_port(options=options)
221         self.assertEqual(port.get_option('foo'), 'bar')
222
223     def test_get_option__unset(self):
224         port = self.make_port()
225         self.assertEqual(port.get_option('foo'), None)
226
227     def test_get_option__default(self):
228         port = self.make_port()
229         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
230
231     def test_name__unset(self):
232         port = self.make_port()
233         self.assertEqual(port.name(), None)
234
235     def test_name__set(self):
236         port = self.make_port(port_name='foo')
237         self.assertEqual(port.name(), 'foo')
238
239     def test_additional_platform_directory(self):
240         filesystem = MockFileSystem()
241         port = self.make_port(port_name='foo', filesystem=filesystem)
242         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
243         layout_test_dir = port.layout_tests_dir()
244         test_file = 'fast/test.html'
245
246         # No additional platform directory
247         self.assertEqual(
248             port.expected_baselines(test_file, '.txt'),
249             [(None, 'fast/test-expected.txt')])
250         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
251
252         # Simple additional platform directory
253         port._options.additional_platform_directory = ['/tmp/local-baselines']
254         filesystem.files = {
255             '/tmp/local-baselines/fast/test-expected.txt': 'foo',
256         }
257         self.assertEqual(
258             port.expected_baselines(test_file, '.txt'),
259             [('/tmp/local-baselines', 'fast/test-expected.txt')])
260         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
261
262         # Multiple additional platform directories
263         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
264         self.assertEqual(
265             port.expected_baselines(test_file, '.txt'),
266             [('/tmp/local-baselines', 'fast/test-expected.txt')])
267         self.assertEqual(port.baseline_path(), '/foo')
268
269     def test_uses_test_expectations_file(self):
270         filesystem = MockFileSystem()
271         port = self.make_port(port_name='foo', filesystem=filesystem)
272         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
273         self.assertFalse(port.uses_test_expectations_file())
274         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
275         self.assertTrue(port.uses_test_expectations_file())
276
277
278 class VirtualTest(unittest.TestCase):
279     """Tests that various methods expected to be virtual are."""
280     def assertVirtual(self, method, *args, **kwargs):
281         self.assertRaises(NotImplementedError, method, *args, **kwargs)
282
283     def test_virtual_methods(self):
284         port = Port(MockHost())
285         self.assertVirtual(port.baseline_path)
286         self.assertVirtual(port.baseline_search_path)
287         self.assertVirtual(port.check_build, None)
288         self.assertVirtual(port.check_image_diff)
289         self.assertVirtual(port.create_driver, 0)
290         self.assertVirtual(port.diff_image, None, None)
291         self.assertVirtual(port.path_to_test_expectations_file)
292         self.assertVirtual(port.default_results_directory)
293         self.assertVirtual(port.test_expectations)
294         self.assertVirtual(port._path_to_apache)
295         self.assertVirtual(port._path_to_apache_config_file)
296         self.assertVirtual(port._path_to_driver)
297         self.assertVirtual(port._path_to_helper)
298         self.assertVirtual(port._path_to_image_diff)
299         self.assertVirtual(port._path_to_lighttpd)
300         self.assertVirtual(port._path_to_lighttpd_modules)
301         self.assertVirtual(port._path_to_lighttpd_php)
302         self.assertVirtual(port._path_to_wdiff)
303
304     def test_virtual_driver_methods(self):
305         driver = Driver(Port(MockHost()), None, pixel_tests=False)
306         self.assertVirtual(driver.run_test, None)
307         self.assertVirtual(driver.stop)
308         self.assertVirtual(driver.cmd_line)
309
310
311 # FIXME: This should be moved to driver_unittest.py or just deleted.
312 class DriverTest(unittest.TestCase):
313
314     def _assert_wrapper(self, wrapper_string, expected_wrapper):
315         wrapper = Driver(Port(MockHost()), None, pixel_tests=False)._command_wrapper(wrapper_string)
316         self.assertEqual(wrapper, expected_wrapper)
317
318     def test_command_wrapper(self):
319         self._assert_wrapper(None, [])
320         self._assert_wrapper("valgrind", ["valgrind"])
321
322         # Validate that shlex works as expected.
323         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
324         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
325         self._assert_wrapper(command_with_spaces, expected_parse)
326
327
328 if __name__ == '__main__':
329     unittest.main()