Tools/Scripts: eliminate find_test_files from Port class.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import optparse
30 import sys
31 import tempfile
32 import unittest
33
34 from webkitpy.common.system.executive import Executive, ScriptError
35 from webkitpy.common.system import executive_mock
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system import outputcapture
38 from webkitpy.common.system.path import abspath_to_uri
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.mocktool import MockOptions
41 from webkitpy.common.system.executive_mock import MockExecutive
42 from webkitpy.common.host_mock import MockHost
43
44 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
45 from webkitpy.layout_tests.port.test import TestPort
46
47 import config
48 import config_mock
49
50
51 class PortTest(unittest.TestCase):
52     def make_port(self, host=None, **kwargs):
53         host = host or MockHost()
54         return Port(host, **kwargs)
55
56     def test_format_wdiff_output_as_html(self):
57         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
58         html = self.make_port()._format_wdiff_output_as_html(output)
59         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
60         self.assertEqual(html, expected_html)
61
62     def test_wdiff_command(self):
63         port = self.make_port()
64         port._path_to_wdiff = lambda: "/path/to/wdiff"
65         command = port._wdiff_command("/actual/path", "/expected/path")
66         expected_command = [
67             "/path/to/wdiff",
68             "--start-delete=##WDIFF_DEL##",
69             "--end-delete=##WDIFF_END##",
70             "--start-insert=##WDIFF_ADD##",
71             "--end-insert=##WDIFF_END##",
72             "/actual/path",
73             "/expected/path",
74         ]
75         self.assertEqual(command, expected_command)
76
77     def _file_with_contents(self, contents, encoding="utf-8"):
78         new_file = tempfile.NamedTemporaryFile()
79         new_file.write(contents.encode(encoding))
80         new_file.flush()
81         return new_file
82
83     def test_pretty_patch_os_error(self):
84         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
85         oc = outputcapture.OutputCapture()
86         oc.capture_output()
87         self.assertEqual(port.pretty_patch_text("patch.txt"),
88                          port._pretty_patch_error_html)
89
90         # This tests repeated calls to make sure we cache the result.
91         self.assertEqual(port.pretty_patch_text("patch.txt"),
92                          port._pretty_patch_error_html)
93         oc.restore_output()
94
95     def test_pretty_patch_script_error(self):
96         # FIXME: This is some ugly white-box test hacking ...
97         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
98         port._pretty_patch_available = True
99         self.assertEqual(port.pretty_patch_text("patch.txt"),
100                          port._pretty_patch_error_html)
101
102         # This tests repeated calls to make sure we cache the result.
103         self.assertEqual(port.pretty_patch_text("patch.txt"),
104                          port._pretty_patch_error_html)
105
106     def integration_test_run_wdiff(self):
107         executive = Executive()
108         # This may fail on some systems.  We could ask the port
109         # object for the wdiff path, but since we don't know what
110         # port object to use, this is sufficient for now.
111         try:
112             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
113         except Exception, e:
114             wdiff_path = None
115
116         port = self.make_port(executive=executive)
117         port._path_to_wdiff = lambda: wdiff_path
118
119         if wdiff_path:
120             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
121             actual = self._file_with_contents(u"foo")
122             expected = self._file_with_contents(u"bar")
123             wdiff = port._run_wdiff(actual.name, expected.name)
124             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
125             self.assertEqual(wdiff, expected_wdiff)
126             # Running the full wdiff_text method should give the same result.
127             port._wdiff_available = True  # In case it's somehow already disabled.
128             wdiff = port.wdiff_text(actual.name, expected.name)
129             self.assertEqual(wdiff, expected_wdiff)
130             # wdiff should still be available after running wdiff_text with a valid diff.
131             self.assertTrue(port._wdiff_available)
132             actual.close()
133             expected.close()
134
135             # Bogus paths should raise a script error.
136             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
137             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
138             # wdiff will still be available after running wdiff_text with invalid paths.
139             self.assertTrue(port._wdiff_available)
140
141         # If wdiff does not exist _run_wdiff should throw an OSError.
142         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
143         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
144
145         # wdiff_text should not throw an error if wdiff does not exist.
146         self.assertEqual(port.wdiff_text("foo", "bar"), "")
147         # However wdiff should not be available after running wdiff_text if wdiff is missing.
148         self.assertFalse(port._wdiff_available)
149
150     def test_diff_text(self):
151         port = self.make_port()
152         # Make sure that we don't run into decoding exceptions when the
153         # filenames are unicode, with regular or malformed input (expected or
154         # actual input is always raw bytes, not unicode).
155         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
156         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
157         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
158
159         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
160         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
161
162         # Though expected and actual files should always be read in with no
163         # encoding (and be stored as str objects), test unicode inputs just to
164         # be safe.
165         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
166         port.diff_text(
167             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
168
169         # And make sure we actually get diff output.
170         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
171         self.assertTrue('foo' in diff)
172         self.assertTrue('bar' in diff)
173         self.assertTrue('exp.txt' in diff)
174         self.assertTrue('act.txt' in diff)
175         self.assertFalse('nosuchthing' in diff)
176
177     def test_default_configuration_notfound(self):
178         # Test that we delegate to the config object properly.
179         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
180         self.assertEqual(port.default_configuration(), 'default')
181
182     def test_layout_tests_skipping(self):
183         filesystem = MockFileSystem({
184             '/mock-checkout/LayoutTests/media/video-zoom.html': '',
185             '/mock-checkout/LayoutTests/foo/bar.html': '',
186         })
187         port = self.make_port(filesystem=filesystem)
188         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
189         self.assertTrue(port.skips_layout_test('foo/bar.html'))
190         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
191         self.assertFalse(port.skips_layout_test('foo/foo.html'))
192
193     def test_setup_test_run(self):
194         port = self.make_port()
195         # This routine is a no-op. We just test it for coverage.
196         port.setup_test_run()
197
198     def test_test_dirs(self):
199         filesystem = MockFileSystem({
200             '/mock-checkout/LayoutTests/canvas/test': '',
201             '/mock-checkout/LayoutTests/css2.1/test': '',
202         })
203         port = self.make_port(filesystem=filesystem)
204         dirs = port.test_dirs()
205         self.assertTrue('canvas' in dirs)
206         self.assertTrue('css2.1' in dirs)
207
208     def test_test_to_uri(self):
209         port = self.make_port()
210         layout_test_dir = port.layout_tests_dir()
211         test = 'foo/bar.html'
212         path = port._filesystem.join(layout_test_dir, test)
213         if sys.platform == 'win32':
214             path = path.replace("\\", "/")
215
216         self.assertEqual(port.test_to_uri(test), abspath_to_uri(path))
217
218     def test_get_option__set(self):
219         options, args = optparse.OptionParser().parse_args([])
220         options.foo = 'bar'
221         port = self.make_port(options=options)
222         self.assertEqual(port.get_option('foo'), 'bar')
223
224     def test_get_option__unset(self):
225         port = self.make_port()
226         self.assertEqual(port.get_option('foo'), None)
227
228     def test_get_option__default(self):
229         port = self.make_port()
230         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
231
232     def test_name__unset(self):
233         port = self.make_port()
234         self.assertEqual(port.name(), None)
235
236     def test_name__set(self):
237         port = self.make_port(port_name='foo')
238         self.assertEqual(port.name(), 'foo')
239
240     def test_additional_platform_directory(self):
241         filesystem = MockFileSystem()
242         port = self.make_port(port_name='foo', filesystem=filesystem)
243         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
244         layout_test_dir = port.layout_tests_dir()
245         test_file = 'fast/test.html'
246
247         # No additional platform directory
248         self.assertEqual(
249             port.expected_baselines(test_file, '.txt'),
250             [(None, 'fast/test-expected.txt')])
251         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
252
253         # Simple additional platform directory
254         port._options.additional_platform_directory = ['/tmp/local-baselines']
255         filesystem.files = {
256             '/tmp/local-baselines/fast/test-expected.txt': 'foo',
257         }
258         self.assertEqual(
259             port.expected_baselines(test_file, '.txt'),
260             [('/tmp/local-baselines', 'fast/test-expected.txt')])
261         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
262
263         # Multiple additional platform directories
264         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
265         self.assertEqual(
266             port.expected_baselines(test_file, '.txt'),
267             [('/tmp/local-baselines', 'fast/test-expected.txt')])
268         self.assertEqual(port.baseline_path(), '/foo')
269
270     def test_uses_test_expectations_file(self):
271         filesystem = MockFileSystem()
272         port = self.make_port(port_name='foo', filesystem=filesystem)
273         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
274         self.assertFalse(port.uses_test_expectations_file())
275         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
276         self.assertTrue(port.uses_test_expectations_file())
277
278     def test_find_no_paths_specified(self):
279         port = TestPort()
280         layout_tests_dir = port.layout_tests_dir()
281         tests = port.tests([])
282         self.assertNotEqual(len(tests), 0)
283
284     def test_find_one_test(self):
285         port = TestPort()
286         tests = port.tests(['failures/expected/image.html'])
287         self.assertEqual(len(tests), 1)
288
289     def test_find_glob(self):
290         port = TestPort()
291         tests = port.tests(['failures/expected/im*'])
292         self.assertEqual(len(tests), 2)
293
294     def test_find_with_skipped_directories(self):
295         port = TestPort()
296         tests = port.tests('userscripts')
297         self.assertTrue('userscripts/resources/iframe.html' not in tests)
298
299     def test_find_with_skipped_directories_2(self):
300         port = TestPort()
301         tests = port.tests(['userscripts/resources'])
302         self.assertEqual(tests, set([]))
303
304     def test_is_test_file(self):
305         filesystem = MockFileSystem()
306         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.html'))
307         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.shtml'))
308         self.assertTrue(Port._is_test_file(filesystem, '', 'test-ref-test.html'))
309         self.assertFalse(Port._is_test_file(filesystem, '', 'foo.png'))
310         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.html'))
311         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
312         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.html'))
313         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.html'))
314         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.xht'))
315         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.xhtml'))
316         self.assertFalse(Port._is_test_file(filesystem, '', 'ref-foo.html'))
317         self.assertFalse(Port._is_test_file(filesystem, '', 'notref-foo.xhr'))
318
319     def test_parse_reftest_list(self):
320         port = TestPort()
321         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
322         "",
323         "# some comment",
324         "!= test-2.html test-notref.html # more comments",
325         "== test-3.html test-ref.html"])
326
327         reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
328         self.assertEqual(reftest_list, {'bar/test.html': ('==', 'bar/test-ref.html'),
329             'bar/test-2.html': ('!=', 'bar/test-notref.html'),
330             'bar/test-3.html': ('==', 'bar/test-ref.html')})
331
332
333 class VirtualTest(unittest.TestCase):
334     """Tests that various methods expected to be virtual are."""
335     def assertVirtual(self, method, *args, **kwargs):
336         self.assertRaises(NotImplementedError, method, *args, **kwargs)
337
338     def test_virtual_methods(self):
339         port = Port(MockHost())
340         self.assertVirtual(port.baseline_path)
341         self.assertVirtual(port.baseline_search_path)
342         self.assertVirtual(port.check_build, None)
343         self.assertVirtual(port.check_image_diff)
344         self.assertVirtual(port.create_driver, 0)
345         self.assertVirtual(port.diff_image, None, None)
346         self.assertVirtual(port.path_to_test_expectations_file)
347         self.assertVirtual(port.default_results_directory)
348         self.assertVirtual(port.test_expectations)
349         self.assertVirtual(port._path_to_apache)
350         self.assertVirtual(port._path_to_apache_config_file)
351         self.assertVirtual(port._path_to_driver)
352         self.assertVirtual(port._path_to_helper)
353         self.assertVirtual(port._path_to_image_diff)
354         self.assertVirtual(port._path_to_lighttpd)
355         self.assertVirtual(port._path_to_lighttpd_modules)
356         self.assertVirtual(port._path_to_lighttpd_php)
357         self.assertVirtual(port._path_to_wdiff)
358
359     def test_virtual_driver_methods(self):
360         driver = Driver(Port(MockHost()), None, pixel_tests=False)
361         self.assertVirtual(driver.run_test, None)
362         self.assertVirtual(driver.stop)
363         self.assertVirtual(driver.cmd_line)
364
365
366 # FIXME: This should be moved to driver_unittest.py or just deleted.
367 class DriverTest(unittest.TestCase):
368
369     def _assert_wrapper(self, wrapper_string, expected_wrapper):
370         wrapper = Driver(Port(MockHost()), None, pixel_tests=False)._command_wrapper(wrapper_string)
371         self.assertEqual(wrapper, expected_wrapper)
372
373     def test_command_wrapper(self):
374         self._assert_wrapper(None, [])
375         self._assert_wrapper("valgrind", ["valgrind"])
376
377         # Validate that shlex works as expected.
378         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
379         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
380         self._assert_wrapper(command_with_spaces, expected_parse)
381
382
383 if __name__ == '__main__':
384     unittest.main()