359053308126e8d9288c4d83cc196dc7653d89c1
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import optparse
30 import sys
31 import tempfile
32 import unittest
33
34 from webkitpy.common.system.executive import Executive, ScriptError
35 from webkitpy.common.system import executive_mock
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system import outputcapture
38 from webkitpy.common.system.path import abspath_to_uri
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.mocktool import MockOptions
41 from webkitpy.common.system.executive_mock import MockExecutive
42 from webkitpy.common.host_mock import MockHost
43
44 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
45 from webkitpy.layout_tests.port.base import _is_test_file
46 from webkitpy.layout_tests.port.base import _parse_reftest_list
47 from webkitpy.layout_tests.port.test import TestPort
48
49 import config
50 import config_mock
51
52
53 class PortTest(unittest.TestCase):
54     def make_port(self, host=None, **kwargs):
55         host = host or MockHost()
56         return Port(host, **kwargs)
57
58     def test_format_wdiff_output_as_html(self):
59         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
60         html = self.make_port()._format_wdiff_output_as_html(output)
61         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
62         self.assertEqual(html, expected_html)
63
64     def test_wdiff_command(self):
65         port = self.make_port()
66         port._path_to_wdiff = lambda: "/path/to/wdiff"
67         command = port._wdiff_command("/actual/path", "/expected/path")
68         expected_command = [
69             "/path/to/wdiff",
70             "--start-delete=##WDIFF_DEL##",
71             "--end-delete=##WDIFF_END##",
72             "--start-insert=##WDIFF_ADD##",
73             "--end-insert=##WDIFF_END##",
74             "/actual/path",
75             "/expected/path",
76         ]
77         self.assertEqual(command, expected_command)
78
79     def _file_with_contents(self, contents, encoding="utf-8"):
80         new_file = tempfile.NamedTemporaryFile()
81         new_file.write(contents.encode(encoding))
82         new_file.flush()
83         return new_file
84
85     def test_pretty_patch_os_error(self):
86         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
87         oc = outputcapture.OutputCapture()
88         oc.capture_output()
89         self.assertEqual(port.pretty_patch_text("patch.txt"),
90                          port._pretty_patch_error_html)
91
92         # This tests repeated calls to make sure we cache the result.
93         self.assertEqual(port.pretty_patch_text("patch.txt"),
94                          port._pretty_patch_error_html)
95         oc.restore_output()
96
97     def test_pretty_patch_script_error(self):
98         # FIXME: This is some ugly white-box test hacking ...
99         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
100         port._pretty_patch_available = True
101         self.assertEqual(port.pretty_patch_text("patch.txt"),
102                          port._pretty_patch_error_html)
103
104         # This tests repeated calls to make sure we cache the result.
105         self.assertEqual(port.pretty_patch_text("patch.txt"),
106                          port._pretty_patch_error_html)
107
108     def integration_test_run_wdiff(self):
109         executive = Executive()
110         # This may fail on some systems.  We could ask the port
111         # object for the wdiff path, but since we don't know what
112         # port object to use, this is sufficient for now.
113         try:
114             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
115         except Exception, e:
116             wdiff_path = None
117
118         port = self.make_port(executive=executive)
119         port._path_to_wdiff = lambda: wdiff_path
120
121         if wdiff_path:
122             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
123             actual = self._file_with_contents(u"foo")
124             expected = self._file_with_contents(u"bar")
125             wdiff = port._run_wdiff(actual.name, expected.name)
126             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
127             self.assertEqual(wdiff, expected_wdiff)
128             # Running the full wdiff_text method should give the same result.
129             port._wdiff_available = True  # In case it's somehow already disabled.
130             wdiff = port.wdiff_text(actual.name, expected.name)
131             self.assertEqual(wdiff, expected_wdiff)
132             # wdiff should still be available after running wdiff_text with a valid diff.
133             self.assertTrue(port._wdiff_available)
134             actual.close()
135             expected.close()
136
137             # Bogus paths should raise a script error.
138             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
139             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
140             # wdiff will still be available after running wdiff_text with invalid paths.
141             self.assertTrue(port._wdiff_available)
142
143         # If wdiff does not exist _run_wdiff should throw an OSError.
144         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
145         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
146
147         # wdiff_text should not throw an error if wdiff does not exist.
148         self.assertEqual(port.wdiff_text("foo", "bar"), "")
149         # However wdiff should not be available after running wdiff_text if wdiff is missing.
150         self.assertFalse(port._wdiff_available)
151
152     def test_diff_text(self):
153         port = self.make_port()
154         # Make sure that we don't run into decoding exceptions when the
155         # filenames are unicode, with regular or malformed input (expected or
156         # actual input is always raw bytes, not unicode).
157         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
158         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
159         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
160
161         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
162         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
163
164         # Though expected and actual files should always be read in with no
165         # encoding (and be stored as str objects), test unicode inputs just to
166         # be safe.
167         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
168         port.diff_text(
169             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
170
171         # And make sure we actually get diff output.
172         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
173         self.assertTrue('foo' in diff)
174         self.assertTrue('bar' in diff)
175         self.assertTrue('exp.txt' in diff)
176         self.assertTrue('act.txt' in diff)
177         self.assertFalse('nosuchthing' in diff)
178
179     def test_default_configuration_notfound(self):
180         # Test that we delegate to the config object properly.
181         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
182         self.assertEqual(port.default_configuration(), 'default')
183
184     def test_layout_tests_skipping(self):
185         filesystem = MockFileSystem({
186             '/mock-checkout/LayoutTests/media/video-zoom.html': '',
187             '/mock-checkout/LayoutTests/foo/bar.html': '',
188         })
189         port = self.make_port(filesystem=filesystem)
190         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
191         self.assertTrue(port.skips_layout_test('foo/bar.html'))
192         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
193         self.assertFalse(port.skips_layout_test('foo/foo.html'))
194
195     def test_setup_test_run(self):
196         port = self.make_port()
197         # This routine is a no-op. We just test it for coverage.
198         port.setup_test_run()
199
200     def test_test_dirs(self):
201         filesystem = MockFileSystem({
202             '/mock-checkout/LayoutTests/canvas/test': '',
203             '/mock-checkout/LayoutTests/css2.1/test': '',
204         })
205         port = self.make_port(filesystem=filesystem)
206         dirs = port.test_dirs()
207         self.assertTrue('canvas' in dirs)
208         self.assertTrue('css2.1' in dirs)
209
210     def test_test_to_uri(self):
211         port = self.make_port()
212         layout_test_dir = port.layout_tests_dir()
213         test = 'foo/bar.html'
214         path = port._filesystem.join(layout_test_dir, test)
215         if sys.platform == 'win32':
216             path = path.replace("\\", "/")
217
218         self.assertEqual(port.test_to_uri(test), abspath_to_uri(path))
219
220     def test_get_option__set(self):
221         options, args = optparse.OptionParser().parse_args([])
222         options.foo = 'bar'
223         port = self.make_port(options=options)
224         self.assertEqual(port.get_option('foo'), 'bar')
225
226     def test_get_option__unset(self):
227         port = self.make_port()
228         self.assertEqual(port.get_option('foo'), None)
229
230     def test_get_option__default(self):
231         port = self.make_port()
232         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
233
234     def test_name__unset(self):
235         port = self.make_port()
236         self.assertEqual(port.name(), None)
237
238     def test_name__set(self):
239         port = self.make_port(port_name='foo')
240         self.assertEqual(port.name(), 'foo')
241
242     def test_additional_platform_directory(self):
243         filesystem = MockFileSystem()
244         port = self.make_port(port_name='foo', filesystem=filesystem)
245         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
246         layout_test_dir = port.layout_tests_dir()
247         test_file = 'fast/test.html'
248
249         # No additional platform directory
250         self.assertEqual(
251             port.expected_baselines(test_file, '.txt'),
252             [(None, 'fast/test-expected.txt')])
253         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
254
255         # Simple additional platform directory
256         port._options.additional_platform_directory = ['/tmp/local-baselines']
257         filesystem.files = {
258             '/tmp/local-baselines/fast/test-expected.txt': 'foo',
259         }
260         self.assertEqual(
261             port.expected_baselines(test_file, '.txt'),
262             [('/tmp/local-baselines', 'fast/test-expected.txt')])
263         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
264
265         # Multiple additional platform directories
266         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
267         self.assertEqual(
268             port.expected_baselines(test_file, '.txt'),
269             [('/tmp/local-baselines', 'fast/test-expected.txt')])
270         self.assertEqual(port.baseline_path(), '/foo')
271
272     def test_uses_test_expectations_file(self):
273         filesystem = MockFileSystem()
274         port = self.make_port(port_name='foo', filesystem=filesystem)
275         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
276         self.assertFalse(port.uses_test_expectations_file())
277         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
278         self.assertTrue(port.uses_test_expectations_file())
279
280     def test_find_no_paths_specified(self):
281         port = TestPort()
282         layout_tests_dir = port.layout_tests_dir()
283         tests = port.find_test_files([])
284         self.assertNotEqual(len(tests), 0)
285
286     def test_find_one_test(self):
287         port = TestPort()
288         tests = port.find_test_files(['failures/expected/image.html'])
289         self.assertEqual(len(tests), 1)
290
291     def test_find_glob(self):
292         port = TestPort()
293         tests = port.find_test_files(['failures/expected/im*'])
294         self.assertEqual(len(tests), 2)
295
296     def test_find_with_skipped_directories(self):
297         port = TestPort()
298         tests = port.find_test_files('userscripts')
299         self.assertTrue('userscripts/resources/iframe.html' not in tests)
300
301     def test_find_with_skipped_directories_2(self):
302         port = TestPort()
303         tests = port.find_test_files(['userscripts/resources'])
304         self.assertEqual(tests, set([]))
305
306     def test_is_test_file(self):
307         filesystem = MockFileSystem()
308         self.assertTrue(_is_test_file(filesystem, '', 'foo.html'))
309         self.assertTrue(_is_test_file(filesystem, '', 'foo.shtml'))
310         self.assertTrue(_is_test_file(filesystem, '', 'test-ref-test.html'))
311         self.assertFalse(_is_test_file(filesystem, '', 'foo.png'))
312         self.assertFalse(_is_test_file(filesystem, '', 'foo-expected.html'))
313         self.assertFalse(_is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
314         self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.html'))
315         self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.html'))
316         self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.xht'))
317         self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.xhtml'))
318         self.assertFalse(_is_test_file(filesystem, '', 'ref-foo.html'))
319         self.assertFalse(_is_test_file(filesystem, '', 'notref-foo.xhr'))
320
321     def test_parse_reftest_list(self):
322         port = TestPort()
323         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
324         "",
325         "# some comment",
326         "!= test-2.html test-notref.html # more comments",
327         "== test-3.html test-ref.html"])
328
329         reftest_list = _parse_reftest_list(port.host.filesystem, 'bar')
330         self.assertEqual(reftest_list, {'bar/test.html': ('==', 'bar/test-ref.html'),
331             'bar/test-2.html': ('!=', 'bar/test-notref.html'),
332             'bar/test-3.html': ('==', 'bar/test-ref.html')})
333
334 class VirtualTest(unittest.TestCase):
335     """Tests that various methods expected to be virtual are."""
336     def assertVirtual(self, method, *args, **kwargs):
337         self.assertRaises(NotImplementedError, method, *args, **kwargs)
338
339     def test_virtual_methods(self):
340         port = Port(MockHost())
341         self.assertVirtual(port.baseline_path)
342         self.assertVirtual(port.baseline_search_path)
343         self.assertVirtual(port.check_build, None)
344         self.assertVirtual(port.check_image_diff)
345         self.assertVirtual(port.create_driver, 0)
346         self.assertVirtual(port.diff_image, None, None)
347         self.assertVirtual(port.path_to_test_expectations_file)
348         self.assertVirtual(port.default_results_directory)
349         self.assertVirtual(port.test_expectations)
350         self.assertVirtual(port._path_to_apache)
351         self.assertVirtual(port._path_to_apache_config_file)
352         self.assertVirtual(port._path_to_driver)
353         self.assertVirtual(port._path_to_helper)
354         self.assertVirtual(port._path_to_image_diff)
355         self.assertVirtual(port._path_to_lighttpd)
356         self.assertVirtual(port._path_to_lighttpd_modules)
357         self.assertVirtual(port._path_to_lighttpd_php)
358         self.assertVirtual(port._path_to_wdiff)
359
360     def test_virtual_driver_methods(self):
361         driver = Driver(Port(MockHost()), None, pixel_tests=False)
362         self.assertVirtual(driver.run_test, None)
363         self.assertVirtual(driver.stop)
364         self.assertVirtual(driver.cmd_line)
365
366
367 # FIXME: This should be moved to driver_unittest.py or just deleted.
368 class DriverTest(unittest.TestCase):
369
370     def _assert_wrapper(self, wrapper_string, expected_wrapper):
371         wrapper = Driver(Port(MockHost()), None, pixel_tests=False)._command_wrapper(wrapper_string)
372         self.assertEqual(wrapper, expected_wrapper)
373
374     def test_command_wrapper(self):
375         self._assert_wrapper(None, [])
376         self._assert_wrapper("valgrind", ["valgrind"])
377
378         # Validate that shlex works as expected.
379         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
380         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
381         self._assert_wrapper(command_with_spaces, expected_parse)
382
383
384 if __name__ == '__main__':
385     unittest.main()