run-perf-tests ignore Skipped list on chromium
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30 import optparse
31 import sys
32 import tempfile
33 import unittest
34
35 from webkitpy.common.system.executive import Executive, ScriptError
36 from webkitpy.common.system import executive_mock
37 from webkitpy.common.system.filesystem_mock import MockFileSystem
38 from webkitpy.common.system.outputcapture import OutputCapture
39 from webkitpy.common.system.path import abspath_to_uri
40 from webkitpy.thirdparty.mock import Mock
41 from webkitpy.tool.mocktool import MockOptions
42 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
43 from webkitpy.common.system.systemhost_mock import MockSystemHost
44
45 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
46 from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort
47
48 import config
49 import config_mock
50
51 class PortTest(unittest.TestCase):
52     def make_port(self, executive=None, with_tests=False, **kwargs):
53         host = MockSystemHost()
54         if executive:
55             host.executive = executive
56         if with_tests:
57             add_unit_tests_to_mock_filesystem(host.filesystem)
58             return TestPort(host, **kwargs)
59         return Port(host, **kwargs)
60
61     def test_default_child_processes(self):
62         port = self.make_port()
63         # Even though the MockPlatformInfo shows 1GB free memory (enough for 5 DRT instances)
64         # we're still limited by the 2 mock cores we have:
65         self.assertEqual(port.default_child_processes(), 2)
66         bytes_for_drt = 200 * 1024 * 1024
67
68         port.host.platform.free_bytes_memory = lambda: bytes_for_drt
69         expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
70         child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
71         self.assertEqual(child_processes, 1)
72
73         # Make sure that we always use one process, even if we don't have the memory for it.
74         port.host.platform.free_bytes_memory = lambda: bytes_for_drt - 1
75         expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
76         child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
77         self.assertEqual(child_processes, 1)
78
79     def test_format_wdiff_output_as_html(self):
80         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
81         html = self.make_port()._format_wdiff_output_as_html(output)
82         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
83         self.assertEqual(html, expected_html)
84
85     def test_wdiff_command(self):
86         port = self.make_port()
87         port._path_to_wdiff = lambda: "/path/to/wdiff"
88         command = port._wdiff_command("/actual/path", "/expected/path")
89         expected_command = [
90             "/path/to/wdiff",
91             "--start-delete=##WDIFF_DEL##",
92             "--end-delete=##WDIFF_END##",
93             "--start-insert=##WDIFF_ADD##",
94             "--end-insert=##WDIFF_END##",
95             "/actual/path",
96             "/expected/path",
97         ]
98         self.assertEqual(command, expected_command)
99
100     def _file_with_contents(self, contents, encoding="utf-8"):
101         new_file = tempfile.NamedTemporaryFile()
102         new_file.write(contents.encode(encoding))
103         new_file.flush()
104         return new_file
105
106     def test_pretty_patch_os_error(self):
107         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
108         oc = OutputCapture()
109         oc.capture_output()
110         self.assertEqual(port.pretty_patch_text("patch.txt"),
111                          port._pretty_patch_error_html)
112
113         # This tests repeated calls to make sure we cache the result.
114         self.assertEqual(port.pretty_patch_text("patch.txt"),
115                          port._pretty_patch_error_html)
116         oc.restore_output()
117
118     def test_pretty_patch_script_error(self):
119         # FIXME: This is some ugly white-box test hacking ...
120         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
121         port._pretty_patch_available = True
122         self.assertEqual(port.pretty_patch_text("patch.txt"),
123                          port._pretty_patch_error_html)
124
125         # This tests repeated calls to make sure we cache the result.
126         self.assertEqual(port.pretty_patch_text("patch.txt"),
127                          port._pretty_patch_error_html)
128
129     def integration_test_run_wdiff(self):
130         executive = Executive()
131         # This may fail on some systems.  We could ask the port
132         # object for the wdiff path, but since we don't know what
133         # port object to use, this is sufficient for now.
134         try:
135             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
136         except Exception, e:
137             wdiff_path = None
138
139         port = self.make_port(executive=executive)
140         port._path_to_wdiff = lambda: wdiff_path
141
142         if wdiff_path:
143             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
144             actual = self._file_with_contents(u"foo")
145             expected = self._file_with_contents(u"bar")
146             wdiff = port._run_wdiff(actual.name, expected.name)
147             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
148             self.assertEqual(wdiff, expected_wdiff)
149             # Running the full wdiff_text method should give the same result.
150             port._wdiff_available = True  # In case it's somehow already disabled.
151             wdiff = port.wdiff_text(actual.name, expected.name)
152             self.assertEqual(wdiff, expected_wdiff)
153             # wdiff should still be available after running wdiff_text with a valid diff.
154             self.assertTrue(port._wdiff_available)
155             actual.close()
156             expected.close()
157
158             # Bogus paths should raise a script error.
159             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
160             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
161             # wdiff will still be available after running wdiff_text with invalid paths.
162             self.assertTrue(port._wdiff_available)
163
164         # If wdiff does not exist _run_wdiff should throw an OSError.
165         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
166         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
167
168         # wdiff_text should not throw an error if wdiff does not exist.
169         self.assertEqual(port.wdiff_text("foo", "bar"), "")
170         # However wdiff should not be available after running wdiff_text if wdiff is missing.
171         self.assertFalse(port._wdiff_available)
172
173     def test_wdiff_text(self):
174         port = self.make_port()
175         port.wdiff_available = lambda: True
176         port._run_wdiff = lambda a, b: 'PASS'
177         self.assertEqual('PASS', port.wdiff_text(None, None))
178
179     def test_diff_text(self):
180         port = self.make_port()
181         # Make sure that we don't run into decoding exceptions when the
182         # filenames are unicode, with regular or malformed input (expected or
183         # actual input is always raw bytes, not unicode).
184         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
185         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
186         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
187
188         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
189         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
190
191         # Though expected and actual files should always be read in with no
192         # encoding (and be stored as str objects), test unicode inputs just to
193         # be safe.
194         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
195         port.diff_text(
196             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
197
198         # And make sure we actually get diff output.
199         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
200         self.assertTrue('foo' in diff)
201         self.assertTrue('bar' in diff)
202         self.assertTrue('exp.txt' in diff)
203         self.assertTrue('act.txt' in diff)
204         self.assertFalse('nosuchthing' in diff)
205
206     def test_default_configuration_notfound(self):
207         # Test that we delegate to the config object properly.
208         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
209         self.assertEqual(port.default_configuration(), 'default')
210
211     def test_layout_tests_skipping(self):
212         port = self.make_port()
213         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/media/video-zoom.html', '')
214         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/foo/bar.html', '')
215         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
216         self.assertTrue(port.skips_layout_test('foo/bar.html'))
217         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
218         self.assertFalse(port.skips_layout_test('foo/foo.html'))
219
220     def test_setup_test_run(self):
221         port = self.make_port()
222         # This routine is a no-op. We just test it for coverage.
223         port.setup_test_run()
224
225     def test_test_dirs(self):
226         port = self.make_port()
227         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '')
228         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '')
229         dirs = port.test_dirs()
230         self.assertTrue('canvas' in dirs)
231         self.assertTrue('css2.1' in dirs)
232
233     def test_skipped_perf_tests(self):
234         port = self.make_port()
235
236         def add_text_file(dirname, filename, content='some content'):
237             dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname)
238             port.host.filesystem.maybe_make_directory(dirname)
239             port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content)
240
241         add_text_file('inspector', 'test1.html')
242         add_text_file('inspector', 'unsupported_test1.html')
243         add_text_file('inspector', 'test2.html')
244         add_text_file('inspector/resources', 'resource_file.html')
245         add_text_file('unsupported', 'unsupported_test2.html')
246         add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html']))
247         self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html'])
248
249     def test_get_option__set(self):
250         options, args = optparse.OptionParser().parse_args([])
251         options.foo = 'bar'
252         port = self.make_port(options=options)
253         self.assertEqual(port.get_option('foo'), 'bar')
254
255     def test_get_option__unset(self):
256         port = self.make_port()
257         self.assertEqual(port.get_option('foo'), None)
258
259     def test_get_option__default(self):
260         port = self.make_port()
261         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
262
263     def test_additional_platform_directory(self):
264         port = self.make_port(port_name='foo')
265         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
266         layout_test_dir = port.layout_tests_dir()
267         test_file = 'fast/test.html'
268
269         # No additional platform directory
270         self.assertEqual(
271             port.expected_baselines(test_file, '.txt'),
272             [(None, 'fast/test-expected.txt')])
273         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
274
275         # Simple additional platform directory
276         port._options.additional_platform_directory = ['/tmp/local-baselines']
277         port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo')
278         self.assertEqual(
279             port.expected_baselines(test_file, '.txt'),
280             [('/tmp/local-baselines', 'fast/test-expected.txt')])
281         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
282
283         # Multiple additional platform directories
284         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
285         self.assertEqual(
286             port.expected_baselines(test_file, '.txt'),
287             [('/tmp/local-baselines', 'fast/test-expected.txt')])
288         self.assertEqual(port.baseline_path(), '/foo')
289
290     def test_uses_test_expectations_file(self):
291         port = self.make_port(port_name='foo')
292         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
293         self.assertFalse(port.uses_test_expectations_file())
294         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
295         self.assertTrue(port.uses_test_expectations_file())
296
297     def test_find_no_paths_specified(self):
298         port = self.make_port(with_tests=True)
299         layout_tests_dir = port.layout_tests_dir()
300         tests = port.tests([])
301         self.assertNotEqual(len(tests), 0)
302
303     def test_find_one_test(self):
304         port = self.make_port(with_tests=True)
305         tests = port.tests(['failures/expected/image.html'])
306         self.assertEqual(len(tests), 1)
307
308     def test_find_glob(self):
309         port = self.make_port(with_tests=True)
310         tests = port.tests(['failures/expected/im*'])
311         self.assertEqual(len(tests), 2)
312
313     def test_find_with_skipped_directories(self):
314         port = self.make_port(with_tests=True)
315         tests = port.tests('userscripts')
316         self.assertTrue('userscripts/resources/iframe.html' not in tests)
317
318     def test_find_with_skipped_directories_2(self):
319         port = self.make_port(with_tests=True)
320         tests = port.tests(['userscripts/resources'])
321         self.assertEqual(tests, set([]))
322
323     def test_is_test_file(self):
324         filesystem = MockFileSystem()
325         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.html'))
326         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.shtml'))
327         self.assertTrue(Port._is_test_file(filesystem, '', 'test-ref-test.html'))
328         self.assertFalse(Port._is_test_file(filesystem, '', 'foo.png'))
329         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.html'))
330         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
331         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.html'))
332         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.html'))
333         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.xht'))
334         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.xhtml'))
335         self.assertFalse(Port._is_test_file(filesystem, '', 'ref-foo.html'))
336         self.assertFalse(Port._is_test_file(filesystem, '', 'notref-foo.xhr'))
337
338     def test_parse_reftest_list(self):
339         port = self.make_port(with_tests=True)
340         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
341         "",
342         "# some comment",
343         "!= test-2.html test-notref.html # more comments",
344         "== test-3.html test-ref.html",
345         "== test-3.html test-ref2.html",
346         "!= test-3.html test-notref.html"])
347
348         reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
349         self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
350             'bar/test-2.html': [('!=', 'bar/test-notref.html')],
351             'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
352
353     def test_operating_system(self):
354         self.assertEqual('mac', self.make_port().operating_system())
355
356     def test_check_httpd_success(self):
357         port = self.make_port(executive=MockExecutive2())
358         port._path_to_apache = lambda: '/usr/sbin/httpd'
359         capture = OutputCapture()
360         capture.capture_output()
361         self.assertTrue(port.check_httpd())
362         _, _, logs = capture.restore_output()
363         self.assertEqual('', logs)
364
365     def test_httpd_returns_error_code(self):
366         port = self.make_port(executive=MockExecutive2(exit_code=1))
367         port._path_to_apache = lambda: '/usr/sbin/httpd'
368         capture = OutputCapture()
369         capture.capture_output()
370         self.assertFalse(port.check_httpd())
371         _, _, logs = capture.restore_output()
372         self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
373
374     def assertVirtual(self, method, *args, **kwargs):
375         self.assertRaises(NotImplementedError, method, *args, **kwargs)
376
377     def test_virtual_methods(self):
378         port = Port(MockSystemHost())
379         self.assertVirtual(port.baseline_path)
380         self.assertVirtual(port.baseline_search_path)
381         self.assertVirtual(port.check_build, None)
382         self.assertVirtual(port.check_image_diff)
383         self.assertVirtual(port.create_driver, 0)
384         self.assertVirtual(port.diff_image, None, None)
385         self.assertVirtual(port.path_to_test_expectations_file)
386         self.assertVirtual(port.default_results_directory)
387         self.assertVirtual(port.test_expectations)
388         self.assertVirtual(port._path_to_apache)
389         self.assertVirtual(port._path_to_apache_config_file)
390         self.assertVirtual(port._path_to_driver)
391         self.assertVirtual(port._path_to_helper)
392         self.assertVirtual(port._path_to_image_diff)
393         self.assertVirtual(port._path_to_lighttpd)
394         self.assertVirtual(port._path_to_lighttpd_modules)
395         self.assertVirtual(port._path_to_lighttpd_php)
396         self.assertVirtual(port._path_to_wdiff)
397
398
399 if __name__ == '__main__':
400     unittest.main()