Web Inspector: chromium: I'd like to add a script for running perf tests for WebInspe...
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import optparse
30 import sys
31 import tempfile
32 import unittest
33
34 from webkitpy.common.system.executive import Executive, ScriptError
35 from webkitpy.common.system import executive_mock
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system import outputcapture
38 from webkitpy.common.system.path import abspath_to_uri
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.mocktool import MockOptions
41 from webkitpy.common.system.executive_mock import MockExecutive
42 from webkitpy.common.host_mock import MockHost
43
44 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
45 from webkitpy.layout_tests.port.base import _is_test_file
46 from webkitpy.layout_tests.port.test import TestPort
47
48 import config
49 import config_mock
50
51
52 class PortTest(unittest.TestCase):
53     def make_port(self, host=None, **kwargs):
54         host = host or MockHost()
55         return Port(host, **kwargs)
56
57     def test_format_wdiff_output_as_html(self):
58         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
59         html = self.make_port()._format_wdiff_output_as_html(output)
60         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
61         self.assertEqual(html, expected_html)
62
63     def test_wdiff_command(self):
64         port = self.make_port()
65         port._path_to_wdiff = lambda: "/path/to/wdiff"
66         command = port._wdiff_command("/actual/path", "/expected/path")
67         expected_command = [
68             "/path/to/wdiff",
69             "--start-delete=##WDIFF_DEL##",
70             "--end-delete=##WDIFF_END##",
71             "--start-insert=##WDIFF_ADD##",
72             "--end-insert=##WDIFF_END##",
73             "/actual/path",
74             "/expected/path",
75         ]
76         self.assertEqual(command, expected_command)
77
78     def _file_with_contents(self, contents, encoding="utf-8"):
79         new_file = tempfile.NamedTemporaryFile()
80         new_file.write(contents.encode(encoding))
81         new_file.flush()
82         return new_file
83
84     def test_pretty_patch_os_error(self):
85         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
86         oc = outputcapture.OutputCapture()
87         oc.capture_output()
88         self.assertEqual(port.pretty_patch_text("patch.txt"),
89                          port._pretty_patch_error_html)
90
91         # This tests repeated calls to make sure we cache the result.
92         self.assertEqual(port.pretty_patch_text("patch.txt"),
93                          port._pretty_patch_error_html)
94         oc.restore_output()
95
96     def test_pretty_patch_script_error(self):
97         # FIXME: This is some ugly white-box test hacking ...
98         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
99         port._pretty_patch_available = True
100         self.assertEqual(port.pretty_patch_text("patch.txt"),
101                          port._pretty_patch_error_html)
102
103         # This tests repeated calls to make sure we cache the result.
104         self.assertEqual(port.pretty_patch_text("patch.txt"),
105                          port._pretty_patch_error_html)
106
107     def integration_test_run_wdiff(self):
108         executive = Executive()
109         # This may fail on some systems.  We could ask the port
110         # object for the wdiff path, but since we don't know what
111         # port object to use, this is sufficient for now.
112         try:
113             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
114         except Exception, e:
115             wdiff_path = None
116
117         port = self.make_port(executive=executive)
118         port._path_to_wdiff = lambda: wdiff_path
119
120         if wdiff_path:
121             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
122             actual = self._file_with_contents(u"foo")
123             expected = self._file_with_contents(u"bar")
124             wdiff = port._run_wdiff(actual.name, expected.name)
125             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
126             self.assertEqual(wdiff, expected_wdiff)
127             # Running the full wdiff_text method should give the same result.
128             port._wdiff_available = True  # In case it's somehow already disabled.
129             wdiff = port.wdiff_text(actual.name, expected.name)
130             self.assertEqual(wdiff, expected_wdiff)
131             # wdiff should still be available after running wdiff_text with a valid diff.
132             self.assertTrue(port._wdiff_available)
133             actual.close()
134             expected.close()
135
136             # Bogus paths should raise a script error.
137             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
138             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
139             # wdiff will still be available after running wdiff_text with invalid paths.
140             self.assertTrue(port._wdiff_available)
141
142         # If wdiff does not exist _run_wdiff should throw an OSError.
143         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
144         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
145
146         # wdiff_text should not throw an error if wdiff does not exist.
147         self.assertEqual(port.wdiff_text("foo", "bar"), "")
148         # However wdiff should not be available after running wdiff_text if wdiff is missing.
149         self.assertFalse(port._wdiff_available)
150
151     def test_diff_text(self):
152         port = self.make_port()
153         # Make sure that we don't run into decoding exceptions when the
154         # filenames are unicode, with regular or malformed input (expected or
155         # actual input is always raw bytes, not unicode).
156         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
157         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
158         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
159
160         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
161         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
162
163         # Though expected and actual files should always be read in with no
164         # encoding (and be stored as str objects), test unicode inputs just to
165         # be safe.
166         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
167         port.diff_text(
168             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
169
170         # And make sure we actually get diff output.
171         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
172         self.assertTrue('foo' in diff)
173         self.assertTrue('bar' in diff)
174         self.assertTrue('exp.txt' in diff)
175         self.assertTrue('act.txt' in diff)
176         self.assertFalse('nosuchthing' in diff)
177
178     def test_default_configuration_notfound(self):
179         # Test that we delegate to the config object properly.
180         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
181         self.assertEqual(port.default_configuration(), 'default')
182
183     def test_layout_tests_skipping(self):
184         filesystem = MockFileSystem({
185             '/mock-checkout/LayoutTests/media/video-zoom.html': '',
186             '/mock-checkout/LayoutTests/foo/bar.html': '',
187         })
188         port = self.make_port(filesystem=filesystem)
189         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
190         self.assertTrue(port.skips_layout_test('foo/bar.html'))
191         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
192         self.assertFalse(port.skips_layout_test('foo/foo.html'))
193
194     def test_setup_test_run(self):
195         port = self.make_port()
196         # This routine is a no-op. We just test it for coverage.
197         port.setup_test_run()
198
199     def test_test_dirs(self):
200         filesystem = MockFileSystem({
201             '/mock-checkout/LayoutTests/canvas/test': '',
202             '/mock-checkout/LayoutTests/css2.1/test': '',
203         })
204         port = self.make_port(filesystem=filesystem)
205         dirs = port.test_dirs()
206         self.assertTrue('canvas' in dirs)
207         self.assertTrue('css2.1' in dirs)
208
209     def test_test_to_uri(self):
210         port = self.make_port()
211         layout_test_dir = port.layout_tests_dir()
212         test = 'foo/bar.html'
213         path = port._filesystem.join(layout_test_dir, test)
214         if sys.platform == 'win32':
215             path = path.replace("\\", "/")
216
217         self.assertEqual(port.test_to_uri(test), abspath_to_uri(path))
218
219     def test_get_option__set(self):
220         options, args = optparse.OptionParser().parse_args([])
221         options.foo = 'bar'
222         port = self.make_port(options=options)
223         self.assertEqual(port.get_option('foo'), 'bar')
224
225     def test_get_option__unset(self):
226         port = self.make_port()
227         self.assertEqual(port.get_option('foo'), None)
228
229     def test_get_option__default(self):
230         port = self.make_port()
231         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
232
233     def test_name__unset(self):
234         port = self.make_port()
235         self.assertEqual(port.name(), None)
236
237     def test_name__set(self):
238         port = self.make_port(port_name='foo')
239         self.assertEqual(port.name(), 'foo')
240
241     def test_additional_platform_directory(self):
242         filesystem = MockFileSystem()
243         port = self.make_port(port_name='foo', filesystem=filesystem)
244         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
245         layout_test_dir = port.layout_tests_dir()
246         test_file = 'fast/test.html'
247
248         # No additional platform directory
249         self.assertEqual(
250             port.expected_baselines(test_file, '.txt'),
251             [(None, 'fast/test-expected.txt')])
252         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
253
254         # Simple additional platform directory
255         port._options.additional_platform_directory = ['/tmp/local-baselines']
256         filesystem.files = {
257             '/tmp/local-baselines/fast/test-expected.txt': 'foo',
258         }
259         self.assertEqual(
260             port.expected_baselines(test_file, '.txt'),
261             [('/tmp/local-baselines', 'fast/test-expected.txt')])
262         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
263
264         # Multiple additional platform directories
265         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
266         self.assertEqual(
267             port.expected_baselines(test_file, '.txt'),
268             [('/tmp/local-baselines', 'fast/test-expected.txt')])
269         self.assertEqual(port.baseline_path(), '/foo')
270
271     def test_uses_test_expectations_file(self):
272         filesystem = MockFileSystem()
273         port = self.make_port(port_name='foo', filesystem=filesystem)
274         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
275         self.assertFalse(port.uses_test_expectations_file())
276         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
277         self.assertTrue(port.uses_test_expectations_file())
278
279     def test_find_no_paths_specified(self):
280         port = TestPort()
281         layout_tests_dir = port.layout_tests_dir()
282         tests = port.find_test_files([])
283         self.assertNotEqual(len(tests), 0)
284
285     def test_find_one_test(self):
286         port = TestPort()
287         tests = port.find_test_files(['failures/expected/image.html'])
288         self.assertEqual(len(tests), 1)
289
290     def test_find_glob(self):
291         port = TestPort()
292         tests = port.find_test_files(['failures/expected/im*'])
293         self.assertEqual(len(tests), 2)
294
295     def test_find_with_skipped_directories(self):
296         port = TestPort()
297         tests = port.find_test_files('userscripts')
298         self.assertTrue('userscripts/resources/iframe.html' not in tests)
299
300     def test_find_with_skipped_directories_2(self):
301         port = TestPort()
302         tests = port.find_test_files(['userscripts/resources'])
303         self.assertEqual(tests, set([]))
304
305     def test_is_test_file(self):
306         filesystem = MockFileSystem()
307         self.assertTrue(_is_test_file(filesystem, '', 'foo.html'))
308         self.assertTrue(_is_test_file(filesystem, '', 'foo.shtml'))
309         self.assertFalse(_is_test_file(filesystem, '', 'foo.png'))
310         self.assertFalse(_is_test_file(filesystem, '', 'foo-expected.html'))
311         self.assertFalse(_is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
312
313
314 class VirtualTest(unittest.TestCase):
315     """Tests that various methods expected to be virtual are."""
316     def assertVirtual(self, method, *args, **kwargs):
317         self.assertRaises(NotImplementedError, method, *args, **kwargs)
318
319     def test_virtual_methods(self):
320         port = Port(MockHost())
321         self.assertVirtual(port.baseline_path)
322         self.assertVirtual(port.baseline_search_path)
323         self.assertVirtual(port.check_build, None)
324         self.assertVirtual(port.check_image_diff)
325         self.assertVirtual(port.create_driver, 0)
326         self.assertVirtual(port.diff_image, None, None)
327         self.assertVirtual(port.path_to_test_expectations_file)
328         self.assertVirtual(port.default_results_directory)
329         self.assertVirtual(port.test_expectations)
330         self.assertVirtual(port._path_to_apache)
331         self.assertVirtual(port._path_to_apache_config_file)
332         self.assertVirtual(port._path_to_driver)
333         self.assertVirtual(port._path_to_helper)
334         self.assertVirtual(port._path_to_image_diff)
335         self.assertVirtual(port._path_to_lighttpd)
336         self.assertVirtual(port._path_to_lighttpd_modules)
337         self.assertVirtual(port._path_to_lighttpd_php)
338         self.assertVirtual(port._path_to_wdiff)
339
340     def test_virtual_driver_methods(self):
341         driver = Driver(Port(MockHost()), None, pixel_tests=False)
342         self.assertVirtual(driver.run_test, None)
343         self.assertVirtual(driver.stop)
344         self.assertVirtual(driver.cmd_line)
345
346
347 # FIXME: This should be moved to driver_unittest.py or just deleted.
348 class DriverTest(unittest.TestCase):
349
350     def _assert_wrapper(self, wrapper_string, expected_wrapper):
351         wrapper = Driver(Port(MockHost()), None, pixel_tests=False)._command_wrapper(wrapper_string)
352         self.assertEqual(wrapper, expected_wrapper)
353
354     def test_command_wrapper(self):
355         self._assert_wrapper(None, [])
356         self._assert_wrapper("valgrind", ["valgrind"])
357
358         # Validate that shlex works as expected.
359         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
360         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
361         self._assert_wrapper(command_with_spaces, expected_parse)
362
363
364 if __name__ == '__main__':
365     unittest.main()