65e8737b8f0bfed3309d6417a67dea2524a9b12d
[WebKit-https.git] / Tools / Scripts / webkitpy / port / base_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30 import optparse
31 import sys
32 import tempfile
33 import unittest
34
35 from webkitpy.common.system.executive import Executive, ScriptError
36 from webkitpy.common.system import executive_mock
37 from webkitpy.common.system.filesystem_mock import MockFileSystem
38 from webkitpy.common.system.outputcapture import OutputCapture
39 from webkitpy.common.system.path import abspath_to_uri
40 from webkitpy.thirdparty.mock import Mock
41 from webkitpy.tool.mocktool import MockOptions
42 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
43 from webkitpy.common.system.systemhost_mock import MockSystemHost
44
45 from webkitpy.port import Port, Driver, DriverOutput
46 from webkitpy.port.test import add_unit_tests_to_mock_filesystem, TestPort
47
48 class PortTest(unittest.TestCase):
49     def make_port(self, executive=None, with_tests=False, port_name=None, **kwargs):
50         host = MockSystemHost()
51         if executive:
52             host.executive = executive
53         if with_tests:
54             add_unit_tests_to_mock_filesystem(host.filesystem)
55             return TestPort(host, **kwargs)
56         return Port(host, port_name or 'baseport', **kwargs)
57
58     def test_default_child_processes(self):
59         port = self.make_port()
60         self.assertIsNotNone(port.default_child_processes())
61
62     def _file_with_contents(self, contents, encoding="utf-8"):
63         new_file = tempfile.NamedTemporaryFile()
64         new_file.write(contents.encode(encoding))
65         new_file.flush()
66         return new_file
67
68     def test_pretty_patch_os_error(self):
69         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
70         oc = OutputCapture()
71         oc.capture_output()
72         self.assertEqual(port.pretty_patch.pretty_patch_text("patch.txt"),
73                          port.pretty_patch.pretty_patch_error_html)
74
75         # This tests repeated calls to make sure we cache the result.
76         self.assertEqual(port.pretty_patch.pretty_patch_text("patch.txt"),
77                          port.pretty_patch.pretty_patch_error_html)
78         oc.restore_output()
79
80     def test_pretty_patch_script_error(self):
81         # FIXME: This is some ugly white-box test hacking ...
82         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
83         port.pretty_patch.ppatch_available = True
84         self.assertEqual(port.pretty_patch.pretty_patch_text("patch.txt"),
85                          port.pretty_patch.pretty_patch_error_html)
86
87         # This tests repeated calls to make sure we cache the result.
88         self.assertEqual(port.pretty_patch.pretty_patch_text("patch.txt"),
89                          port.pretty_patch.pretty_patch_error_html)
90
91
92     def test_diff_text(self):
93         port = self.make_port()
94         # Make sure that we don't run into decoding exceptions when the
95         # filenames are unicode, with regular or malformed input (expected or
96         # actual input is always raw bytes, not unicode).
97         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
98         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
99         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
100
101         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
102         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
103
104         # Though expected and actual files should always be read in with no
105         # encoding (and be stored as str objects), test unicode inputs just to
106         # be safe.
107         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
108         port.diff_text(
109             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
110
111         t1 = "A\n\nB"
112         t2 = "A\n\nB\n\n\n"
113         t3 = "--- exp.txt\n+++ act.txt\n@@ -1,3 +1,5 @@\n A\n \n-B\n\ No newline at end of file\n+B\n+\n+\n"
114         self.assertEqual(t3, port.diff_text(t1, t2, 'exp.txt', 'act.txt'))
115
116         # And make sure we actually get diff output.
117         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
118         self.assertIn('foo', diff)
119         self.assertIn('bar', diff)
120         self.assertIn('exp.txt', diff)
121         self.assertIn('act.txt', diff)
122         self.assertNotIn('nosuchthing', diff)
123
124     def test_setup_test_run(self):
125         port = self.make_port()
126         # This routine is a no-op. We just test it for coverage.
127         port.setup_test_run()
128
129     def test_test_dirs(self):
130         port = self.make_port()
131         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '')
132         port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '')
133         dirs = port.test_dirs()
134         self.assertIn('canvas', dirs)
135         self.assertIn('css2.1', dirs)
136
137     def test_skipped_perf_tests(self):
138         port = self.make_port()
139
140         def add_text_file(dirname, filename, content='some content'):
141             dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname)
142             port.host.filesystem.maybe_make_directory(dirname)
143             port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content)
144
145         add_text_file('inspector', 'test1.html')
146         add_text_file('inspector', 'unsupported_test1.html')
147         add_text_file('inspector', 'test2.html')
148         add_text_file('inspector/resources', 'resource_file.html')
149         add_text_file('unsupported', 'unsupported_test2.html')
150         add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html']))
151         self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html'])
152
153     def test_get_option__set(self):
154         options, args = optparse.OptionParser().parse_args([])
155         options.foo = 'bar'
156         port = self.make_port(options=options)
157         self.assertEqual(port.get_option('foo'), 'bar')
158
159     def test_get_option__unset(self):
160         port = self.make_port()
161         self.assertIsNone(port.get_option('foo'))
162
163     def test_get_option__default(self):
164         port = self.make_port()
165         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
166
167     def test_additional_platform_directory(self):
168         port = self.make_port(port_name='foo')
169         port.default_baseline_search_path = lambda: ['LayoutTests/platform/foo']
170         layout_test_dir = port.layout_tests_dir()
171         test_file = 'fast/test.html'
172
173         # No additional platform directory
174         self.assertEqual(
175             port.expected_baselines(test_file, '.txt'),
176             [(None, 'fast/test-expected.txt')])
177         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
178
179         # Simple additional platform directory
180         port._options.additional_platform_directory = ['/tmp/local-baselines']
181         port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo')
182         self.assertEqual(
183             port.expected_baselines(test_file, '.txt'),
184             [('/tmp/local-baselines', 'fast/test-expected.txt')])
185         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
186
187         # Multiple additional platform directories
188         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
189         self.assertEqual(
190             port.expected_baselines(test_file, '.txt'),
191             [('/tmp/local-baselines', 'fast/test-expected.txt')])
192         self.assertEqual(port.baseline_path(), '/foo')
193
194     def test_nonexistant_expectations(self):
195         port = self.make_port(port_name='foo')
196         port.expectations_files = lambda: ['/mock-checkout/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/LayoutTests/platform/nonexistant/TestExpectations']
197         port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/exists/TestExpectations', '')
198         self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/LayoutTests/platform/exists/TestExpectations')
199
200     def test_additional_expectations(self):
201         port = self.make_port(port_name='foo')
202         port.port_name = 'foo'
203         port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/foo/TestExpectations', '')
204         port._filesystem.write_text_file(
205             '/tmp/additional-expectations-1.txt', 'content1\n')
206         port._filesystem.write_text_file(
207             '/tmp/additional-expectations-2.txt', 'content2\n')
208
209         self.assertEqual('\n'.join(port.expectations_dict().values()), '')
210
211         port._options.additional_expectations = [
212             '/tmp/additional-expectations-1.txt']
213         self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n')
214
215         port._options.additional_expectations = [
216             '/tmp/nonexistent-file', '/tmp/additional-expectations-1.txt']
217         self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n')
218
219         port._options.additional_expectations = [
220             '/tmp/additional-expectations-1.txt', '/tmp/additional-expectations-2.txt']
221         self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n\ncontent2\n')
222
223     def test_additional_env_var(self):
224         port = self.make_port(options=optparse.Values({'additional_env_var': ['FOO=BAR', 'BAR=FOO']}))
225         self.assertEqual(port.get_option('additional_env_var'), ['FOO=BAR', 'BAR=FOO'])
226         environment = port.setup_environ_for_server()
227         self.assertTrue(('FOO' in environment) & ('BAR' in environment))
228         self.assertEqual(environment['FOO'], 'BAR')
229         self.assertEqual(environment['BAR'], 'FOO')
230
231     def test_uses_test_expectations_file(self):
232         port = self.make_port(port_name='foo')
233         port.port_name = 'foo'
234         port.path_to_test_expectations_file = lambda: '/mock-results/TestExpectations'
235         self.assertFalse(port.uses_test_expectations_file())
236         port._filesystem = MockFileSystem({'/mock-results/TestExpectations': ''})
237         self.assertTrue(port.uses_test_expectations_file())
238
239     def test_find_no_paths_specified(self):
240         port = self.make_port(with_tests=True)
241         layout_tests_dir = port.layout_tests_dir()
242         tests = port.tests([])
243         self.assertNotEqual(len(tests), 0)
244
245     def test_find_one_test(self):
246         port = self.make_port(with_tests=True)
247         tests = port.tests(['failures/expected/image.html'])
248         self.assertEqual(len(tests), 1)
249
250     def test_find_glob(self):
251         port = self.make_port(with_tests=True)
252         tests = port.tests(['failures/expected/im*'])
253         self.assertEqual(len(tests), 2)
254
255     def test_find_with_skipped_directories(self):
256         port = self.make_port(with_tests=True)
257         tests = port.tests(['userscripts'])
258         self.assertNotIn('userscripts/resources/iframe.html', tests)
259
260     def test_find_with_skipped_directories_2(self):
261         port = self.make_port(with_tests=True)
262         tests = port.tests(['userscripts/resources'])
263         self.assertEqual(tests, [])
264
265     def test_is_test_file(self):
266         filesystem = MockFileSystem()
267         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.html'))
268         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.shtml'))
269         self.assertTrue(Port._is_test_file(filesystem, '', 'foo.svg'))
270         self.assertTrue(Port._is_test_file(filesystem, '', 'test-ref-test.html'))
271         self.assertFalse(Port._is_test_file(filesystem, '', 'foo.png'))
272         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.html'))
273         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.svg'))
274         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected.xht'))
275         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
276         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.svg'))
277         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml'))
278         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.html'))
279         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.html'))
280         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-notref.xht'))
281         self.assertFalse(Port._is_test_file(filesystem, '', 'foo-ref.xhtml'))
282         self.assertFalse(Port._is_test_file(filesystem, '', 'ref-foo.html'))
283         self.assertFalse(Port._is_test_file(filesystem, '', 'notref-foo.xhr'))
284
285     def test_is_reference_html_file(self):
286         filesystem = MockFileSystem()
287         self.assertTrue(Port.is_reference_html_file(filesystem, '', 'foo-expected.html'))
288         self.assertTrue(Port.is_reference_html_file(filesystem, '', 'foo-expected-mismatch.xml'))
289         self.assertTrue(Port.is_reference_html_file(filesystem, '', 'foo-ref.xhtml'))
290         self.assertTrue(Port.is_reference_html_file(filesystem, '', 'foo-notref.svg'))
291         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo.html'))
292         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.txt'))
293         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.shtml'))
294         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.php'))
295         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.mht'))
296
297     def test_parse_reftest_list(self):
298         port = self.make_port(with_tests=True)
299         port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
300         "",
301         "# some comment",
302         "!= test-2.html test-notref.html # more comments",
303         "== test-3.html test-ref.html",
304         "== test-3.html test-ref2.html",
305         "!= test-3.html test-notref.html"])
306
307         reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
308         self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
309             'bar/test-2.html': [('!=', 'bar/test-notref.html')],
310             'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
311
312     def test_reference_files(self):
313         port = self.make_port(with_tests=True)
314         self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])
315         self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')])
316         self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')])
317
318     def test_operating_system(self):
319         self.assertEqual('mac', self.make_port().operating_system())
320
321     def test_http_server_supports_ipv6(self):
322         port = self.make_port()
323         self.assertTrue(port.http_server_supports_ipv6())
324         port.host.platform.os_name = 'cygwin'
325         self.assertFalse(port.http_server_supports_ipv6())
326         port.host.platform.os_name = 'win'
327         self.assertTrue(port.http_server_supports_ipv6())
328
329     def test_check_httpd_success(self):
330         port = self.make_port(executive=MockExecutive2())
331         port._path_to_apache = lambda: '/usr/sbin/httpd'
332         capture = OutputCapture()
333         capture.capture_output()
334         self.assertTrue(port.check_httpd())
335         _, _, logs = capture.restore_output()
336         self.assertEqual('', logs)
337
338     def test_httpd_returns_error_code(self):
339         port = self.make_port(executive=MockExecutive2(exit_code=1))
340         port._path_to_apache = lambda: '/usr/sbin/httpd'
341         capture = OutputCapture()
342         capture.capture_output()
343         self.assertFalse(port.check_httpd())
344         _, _, logs = capture.restore_output()
345         self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
346
347     def test_test_exists(self):
348         port = self.make_port(with_tests=True)
349         self.assertTrue(port.test_exists('passes'))
350         self.assertTrue(port.test_exists('passes/text.html'))
351         self.assertFalse(port.test_exists('passes/does_not_exist.html'))
352
353         self.assertTrue(port.test_exists('virtual'))
354         self.assertFalse(port.test_exists('virtual/does_not_exist.html'))
355         self.assertTrue(port.test_exists('virtual/passes/text.html'))
356
357     def test_test_isfile(self):
358         port = self.make_port(with_tests=True)
359         self.assertFalse(port.test_isfile('passes'))
360         self.assertTrue(port.test_isfile('passes/text.html'))
361         self.assertFalse(port.test_isfile('passes/does_not_exist.html'))
362
363         self.assertFalse(port.test_isfile('virtual'))
364         self.assertTrue(port.test_isfile('virtual/passes/text.html'))
365         self.assertFalse(port.test_isfile('virtual/does_not_exist.html'))
366
367     def test_test_isdir(self):
368         port = self.make_port(with_tests=True)
369         self.assertTrue(port.test_isdir('passes'))
370         self.assertFalse(port.test_isdir('passes/text.html'))
371         self.assertFalse(port.test_isdir('passes/does_not_exist.html'))
372         self.assertFalse(port.test_isdir('passes/does_not_exist/'))
373
374         self.assertTrue(port.test_isdir('virtual'))
375         self.assertFalse(port.test_isdir('virtual/does_not_exist.html'))
376         self.assertFalse(port.test_isdir('virtual/does_not_exist/'))
377         self.assertFalse(port.test_isdir('virtual/passes/text.html'))
378
379     def test_tests(self):
380         port = self.make_port(with_tests=True)
381         tests = port.tests([])
382         self.assertIn('passes/text.html', tests)
383         self.assertIn('virtual/passes/text.html', tests)
384
385         tests = port.tests(['passes'])
386         self.assertIn('passes/text.html', tests)
387         self.assertIn('passes/passes/test-virtual-passes.html', tests)
388         self.assertNotIn('virtual/passes/text.html', tests)
389
390         tests = port.tests(['virtual/passes'])
391         self.assertNotIn('passes/text.html', tests)
392         self.assertIn('virtual/passes/test-virtual-passes.html', tests)
393         self.assertIn('virtual/passes/passes/test-virtual-passes.html', tests)
394         self.assertNotIn('virtual/passes/test-virtual-virtual/passes.html', tests)
395         self.assertNotIn('virtual/passes/virtual/passes/test-virtual-passes.html', tests)
396
397     def test_build_path(self):
398         port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'}))
399         if port.get_option('configuration') == 'Debug':
400             self.assertEqual(port._build_path(), '/my-build-directory/Debug')
401         else:
402             self.assertEqual(port._build_path(), '/my-build-directory/Release')
403
404
405 class NaturalCompareTest(unittest.TestCase):
406     def setUp(self):
407         self._port = TestPort(MockSystemHost())
408
409     def assert_cmp(self, x, y, result):
410         self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result)
411
412     def test_natural_compare(self):
413         self.assert_cmp('a', 'a', 0)
414         self.assert_cmp('ab', 'a', 1)
415         self.assert_cmp('a', 'ab', -1)
416         self.assert_cmp('', '', 0)
417         self.assert_cmp('', 'ab', -1)
418         self.assert_cmp('1', '2', -1)
419         self.assert_cmp('2', '1', 1)
420         self.assert_cmp('1', '10', -1)
421         self.assert_cmp('2', '10', -1)
422         self.assert_cmp('foo_1.html', 'foo_2.html', -1)
423         self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
424         self.assert_cmp('foo_1.html', 'foo_10.html', -1)
425         self.assert_cmp('foo_2.html', 'foo_10.html', -1)
426         self.assert_cmp('foo_23.html', 'foo_10.html', 1)
427         self.assert_cmp('foo_23.html', 'foo_100.html', -1)
428
429
430 class KeyCompareTest(unittest.TestCase):
431     def setUp(self):
432         self._port = TestPort(MockSystemHost())
433
434     def assert_cmp(self, x, y, result):
435         self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result)
436
437     def test_test_key(self):
438         self.assert_cmp('/a', '/a', 0)
439         self.assert_cmp('/a', '/b', -1)
440         self.assert_cmp('/a2', '/a10', -1)
441         self.assert_cmp('/a2/foo', '/a10/foo', -1)
442         self.assert_cmp('/a/foo11', '/a/foo2', 1)
443         self.assert_cmp('/ab', '/a/a/b', -1)
444         self.assert_cmp('/a/a/b', '/ab', 1)
445         self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)