2010-12-02 Sam Weinig <sam@webkit.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Unit tests for run_webkit_tests."""
32
33 import codecs
34 import itertools
35 import logging
36 import os
37 import Queue
38 import shutil
39 import sys
40 import tempfile
41 import thread
42 import time
43 import threading
44 import unittest
45
46 from webkitpy.common import array_stream
47 from webkitpy.common.system import outputcapture
48 from webkitpy.common.system import user
49 from webkitpy.layout_tests import port
50 from webkitpy.layout_tests.port import test
51 from webkitpy.layout_tests import run_webkit_tests
52 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
53 from webkitpy.layout_tests.port.test import TestPort, TestDriver
54 from webkitpy.python24.versioning import compare_version
55 from webkitpy.test.skip import skip_if
56
57 from webkitpy.thirdparty.mock import Mock
58
59
60 class MockUser():
61     def __init__(self):
62         self.url = None
63
64     def open_url(self, url):
65         self.url = url
66
67
68 def passing_run(extra_args=None, port_obj=None, record_results=False,
69                 tests_included=False):
70     extra_args = extra_args or []
71     args = ['--print', 'nothing']
72     if not '--platform' in extra_args:
73         args.extend(['--platform', 'test'])
74     if not record_results:
75         args.append('--no-record-results')
76     if not '--child-processes' in extra_args:
77         args.extend(['--worker-model', 'inline'])
78     args.extend(extra_args)
79     if not tests_included:
80         # We use the glob to test that globbing works.
81         args.extend(['passes',
82                      'http/tests',
83                      'websocket/tests',
84                      'failures/expected/*'])
85     options, parsed_args = run_webkit_tests.parse_args(args)
86     if not port_obj:
87         port_obj = port.get(port_name=options.platform, options=options,
88                             user=MockUser())
89     res = run_webkit_tests.run(port_obj, options, parsed_args)
90     return res == 0
91
92
93 def logging_run(extra_args=None, port_obj=None, tests_included=False):
94     extra_args = extra_args or []
95     args = ['--no-record-results']
96     if not '--platform' in extra_args:
97         args.extend(['--platform', 'test'])
98     if not '--child-processes' in extra_args:
99         args.extend(['--worker-model', 'inline'])
100     args.extend(extra_args)
101     if not tests_included:
102         args.extend(['passes',
103                      'http/tests',
104                      'websocket/tests',
105                      'failures/expected/*'])
106
107     oc = outputcapture.OutputCapture()
108     try:
109         oc.capture_output()
110         options, parsed_args = run_webkit_tests.parse_args(args)
111         user = MockUser()
112         if not port_obj:
113             port_obj = port.get(port_name=options.platform, options=options,
114                                 user=user)
115         buildbot_output = array_stream.ArrayStream()
116         regular_output = array_stream.ArrayStream()
117         res = run_webkit_tests.run(port_obj, options, parsed_args,
118                                    buildbot_output=buildbot_output,
119                                    regular_output=regular_output)
120     finally:
121         oc.restore_output()
122     return (res, buildbot_output, regular_output, user)
123
124
125 def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False):
126     extra_args = extra_args or []
127     args = [
128         '--print', 'nothing',
129         '--platform', 'test',
130         '--no-record-results',
131         '--worker-model', 'inline']
132     args.extend(extra_args)
133     if not tests_included:
134         # Not including http tests since they get run out of order (that
135         # behavior has its own test, see test_get_test_file_queue)
136         args.extend(['passes', 'failures'])
137     options, parsed_args = run_webkit_tests.parse_args(args)
138     user = MockUser()
139
140     test_batches = []
141
142     class RecordingTestDriver(TestDriver):
143         def __init__(self, port, worker_number):
144             TestDriver.__init__(self, port, worker_number)
145             self._current_test_batch = None
146
147         def poll(self):
148             # So that we don't create a new driver for every test
149             return None
150
151         def stop(self):
152             self._current_test_batch = None
153
154         def run_test(self, test_input):
155             if self._current_test_batch is None:
156                 self._current_test_batch = []
157                 test_batches.append(self._current_test_batch)
158             test_name = self._port.relative_test_filename(test_input.filename)
159             self._current_test_batch.append(test_name)
160             return TestDriver.run_test(self, test_input)
161
162     class RecordingTestPort(TestPort):
163         def create_driver(self, worker_number):
164             return RecordingTestDriver(self, worker_number)
165
166     recording_port = RecordingTestPort(options=options, user=user)
167     logging_run(extra_args=args, port_obj=recording_port, tests_included=True)
168
169     if flatten_batches:
170         return list(itertools.chain(*test_batches))
171
172     return test_batches
173
174 class MainTest(unittest.TestCase):
175     def test_accelerated_compositing(self):
176         # This just tests that we recognize the command line args
177         self.assertTrue(passing_run(['--accelerated-compositing']))
178         self.assertTrue(passing_run(['--no-accelerated-compositing']))
179
180     def test_accelerated_2d_canvas(self):
181         # This just tests that we recognize the command line args
182         self.assertTrue(passing_run(['--accelerated-2d-canvas']))
183         self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
184
185     def test_basic(self):
186         self.assertTrue(passing_run())
187
188     def test_batch_size(self):
189         batch_tests_run = get_tests_run(['--batch-size', '2'])
190         self.assertEquals(len(batch_tests_run), 9)
191         for batch in batch_tests_run:
192             self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
193
194     def test_child_process_1(self):
195         (res, buildbot_output, regular_output, user) = logging_run(
196              ['--print', 'config', '--child-processes', '1'])
197         self.assertTrue('Running one DumpRenderTree\n'
198                         in regular_output.get())
199
200     def test_child_processes_2(self):
201         (res, buildbot_output, regular_output, user) = logging_run(
202              ['--print', 'config', '--child-processes', '2'])
203         self.assertTrue('Running 2 DumpRenderTrees in parallel\n'
204                         in regular_output.get())
205
206     def test_dryrun(self):
207         batch_tests_run = get_tests_run(['--dry-run'])
208         self.assertEqual(batch_tests_run, [])
209
210         batch_tests_run = get_tests_run(['-n'])
211         self.assertEqual(batch_tests_run, [])
212
213     def test_exception_raised(self):
214         self.assertRaises(ValueError, logging_run,
215             ['failures/expected/exception.html'], tests_included=True)
216
217     def test_full_results_html(self):
218         # FIXME: verify html?
219         self.assertTrue(passing_run(['--full-results-html']))
220
221     def test_help_printing(self):
222         res, out, err, user = logging_run(['--help-printing'])
223         self.assertEqual(res, 0)
224         self.assertTrue(out.empty())
225         self.assertFalse(err.empty())
226
227     def test_hung_thread(self):
228         res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
229                                           'failures/expected/hang.html'],
230                                           tests_included=True)
231         self.assertEqual(res, 0)
232         self.assertFalse(out.empty())
233         self.assertFalse(err.empty())
234
235     def test_keyboard_interrupt(self):
236         # Note that this also tests running a test marked as SKIP if
237         # you specify it explicitly.
238         self.assertRaises(KeyboardInterrupt, logging_run,
239             ['failures/expected/keyboard.html'], tests_included=True)
240
241     def test_last_results(self):
242         passing_run(['--clobber-old-results'], record_results=True)
243         (res, buildbot_output, regular_output, user) = logging_run(
244             ['--print-last-failures'])
245         self.assertEqual(regular_output.get(), ['\n\n'])
246         self.assertEqual(buildbot_output.get(), [])
247
248     def test_lint_test_files(self):
249         # FIXME:  add errors?
250         res, out, err, user = logging_run(['--lint-test-files'],
251                                           tests_included=True)
252         self.assertEqual(res, 0)
253         self.assertTrue(out.empty())
254         self.assertTrue(any(['lint succeeded' in msg for msg in err.get()]))
255
256     def test_no_tests_found(self):
257         res, out, err, user = logging_run(['resources'], tests_included=True)
258         self.assertEqual(res, -1)
259         self.assertTrue(out.empty())
260         self.assertTrue('No tests to run.\n' in err.get())
261
262     def test_no_tests_found_2(self):
263         res, out, err, user = logging_run(['foo'], tests_included=True)
264         self.assertEqual(res, -1)
265         self.assertTrue(out.empty())
266         self.assertTrue('No tests to run.\n' in err.get())
267
268     def test_randomize_order(self):
269         # FIXME: verify order was shuffled
270         self.assertTrue(passing_run(['--randomize-order']))
271
272     def test_run_chunk(self):
273         # Test that we actually select the right chunk
274         all_tests_run = get_tests_run(flatten_batches=True)
275         chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
276         self.assertEquals(all_tests_run[4:8], chunk_tests_run)
277
278         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
279         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
280         chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
281         self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
282
283     def test_run_force(self):
284         # This raises an exception because we run
285         # failures/expected/exception.html, which is normally SKIPped.
286         self.assertRaises(ValueError, logging_run, ['--force'])
287
288     def test_run_part(self):
289         # Test that we actually select the right part
290         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
291         tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
292         self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
293
294         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
295         # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
296         # last part repeats the first two tests).
297         chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
298         self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
299
300     def test_run_singly(self):
301         batch_tests_run = get_tests_run(['--run-singly'])
302         self.assertEqual(len(batch_tests_run), 14)
303         for batch in batch_tests_run:
304             self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
305
306     def test_single_file(self):
307         tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True)
308         self.assertEquals(['passes/text.html'], tests_run)
309
310     def test_test_list(self):
311         filename = tempfile.mktemp()
312         tmpfile = file(filename, mode='w+')
313         tmpfile.write('passes/text.html')
314         tmpfile.close()
315         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True)
316         self.assertEquals(['passes/text.html'], tests_run)
317         os.remove(filename)
318         res, out, err, user = logging_run(['--test-list=%s' % filename],
319                                           tests_included=True)
320         self.assertEqual(res, -1)
321         self.assertFalse(err.empty())
322
323     def test_unexpected_failures(self):
324         # Run tests including the unexpected failures.
325         self._url_opened = None
326         res, out, err, user = logging_run(tests_included=True)
327         self.assertEqual(res, 1)
328         self.assertFalse(out.empty())
329         self.assertFalse(err.empty())
330         self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
331
332     def test_results_directory_absolute(self):
333         # We run a configuration that should fail, to generate output, then
334         # look for what the output results url was.
335
336         tmpdir = tempfile.mkdtemp()
337         res, out, err, user = logging_run(['--results-directory=' + tmpdir],
338                                           tests_included=True)
339         self.assertEqual(user.url, os.path.join(tmpdir, 'results.html'))
340         shutil.rmtree(tmpdir, ignore_errors=True)
341
342     def test_results_directory_default(self):
343         # We run a configuration that should fail, to generate output, then
344         # look for what the output results url was.
345
346         # This is the default location.
347         res, out, err, user = logging_run(tests_included=True)
348         self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
349
350     def test_results_directory_relative(self):
351         # We run a configuration that should fail, to generate output, then
352         # look for what the output results url was.
353
354         res, out, err, user = logging_run(['--results-directory=foo'],
355                                           tests_included=True)
356         self.assertEqual(user.url, '/tmp/foo/results.html')
357
358     def test_tolerance(self):
359         class ImageDiffTestPort(TestPort):
360             def diff_image(self, expected_contents, actual_contents,
361                    diff_filename=None):
362                 self.tolerance_used_for_diff_image = self._options.tolerance
363                 return True
364
365         def get_port_for_run(args):
366             options, parsed_args = run_webkit_tests.parse_args(args)
367             test_port = ImageDiffTestPort(options=options, user=MockUser())
368             passing_run(args, port_obj=test_port, tests_included=True)
369             return test_port
370
371         base_args = ['--pixel-tests', 'failures/expected/*']
372
373         # If we pass in an explicit tolerance argument, then that will be used.
374         test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
375         self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
376         test_port = get_port_for_run(base_args + ['--tolerance', '0'])
377         self.assertEqual(0, test_port.tolerance_used_for_diff_image)
378
379         # Otherwise the port's default tolerance behavior (including ignoring it)
380         # should be used.
381         test_port = get_port_for_run(base_args)
382         self.assertEqual(None, test_port.tolerance_used_for_diff_image)
383
384     def test_worker_model__inline(self):
385         self.assertTrue(passing_run(['--worker-model', 'inline']))
386
387     def test_worker_model__threads(self):
388         self.assertTrue(passing_run(['--worker-model', 'threads']))
389
390     def test_worker_model__processes(self):
391         self.assertRaises(ValueError, logging_run,
392                           ['--worker-model', 'processes'])
393
394     def test_worker_model__unknown(self):
395         self.assertRaises(ValueError, logging_run,
396                           ['--worker-model', 'unknown'])
397
398     def test_worker_model__port_override(self):
399         class OverridePort(test.TestPort):
400             def default_worker_model(self):
401                 return 'INVALID'
402
403         options, parsed_args = run_webkit_tests.parse_args(
404             ['--print', 'nothing', '--platform', 'test', '--noshow-results'])
405         port_obj = OverridePort(options=options)
406         self.assertRaises(ValueError, run_webkit_tests.run, port_obj,
407                           options, parsed_args)
408
409
410 MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
411
412
413
414 def _mocked_open(original_open, file_list):
415     def _wrapper(name, mode, encoding):
416         if name.find("-expected.") != -1 and mode.find("w") != -1:
417             # we don't want to actually write new baselines, so stub these out
418             name.replace('\\', '/')
419             file_list.append(name)
420             return original_open(os.devnull, mode, encoding)
421         return original_open(name, mode, encoding)
422     return _wrapper
423
424
425 class RebaselineTest(unittest.TestCase):
426     def assertBaselines(self, file_list, file):
427         "assert that the file_list contains the baselines."""
428         for ext in [".txt", ".png", ".checksum"]:
429             baseline = file + "-expected" + ext
430             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
431
432     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
433     # supposed to be.
434
435     def disabled_test_reset_results(self):
436         # FIXME: This test is disabled until we can rewrite it to use a
437         # mock filesystem.
438         #
439         # Test that we update expectations in place. If the expectation
440         # is missing, update the expected generic location.
441         file_list = []
442         passing_run(['--pixel-tests',
443                         '--reset-results',
444                         'passes/image.html',
445                         'failures/expected/missing_image.html'],
446                         tests_included=True)
447         self.assertEqual(len(file_list), 6)
448         self.assertBaselines(file_list,
449             "data/passes/image")
450         self.assertBaselines(file_list,
451             "data/failures/expected/missing_image")
452
453     def disabled_test_new_baseline(self):
454         # FIXME: This test is disabled until we can rewrite it to use a
455         # mock filesystem.
456         #
457         # Test that we update the platform expectations. If the expectation
458         # is mssing, then create a new expectation in the platform dir.
459         file_list = []
460         original_open = codecs.open
461         try:
462             # Test that we update the platform expectations. If the expectation
463             # is mssing, then create a new expectation in the platform dir.
464             file_list = []
465             codecs.open = _mocked_open(original_open, file_list)
466             passing_run(['--pixel-tests',
467                          '--new-baseline',
468                          'passes/image.html',
469                          'failures/expected/missing_image.html'],
470                         tests_included=True)
471             self.assertEqual(len(file_list), 6)
472             self.assertBaselines(file_list,
473                 "data/platform/test/passes/image")
474             self.assertBaselines(file_list,
475                 "data/platform/test/failures/expected/missing_image")
476         finally:
477             codecs.open = original_open
478
479
480 class TestRunnerWrapper(run_webkit_tests.TestRunner):
481     def _get_test_input_for_file(self, test_file):
482         return test_file
483
484
485 class TestRunnerTest(unittest.TestCase):
486     def test_results_html(self):
487         mock_port = Mock()
488         mock_port.relative_test_filename = lambda name: name
489         mock_port.filename_to_uri = lambda name: name
490
491         runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(),
492             printer=Mock(), message_broker=Mock())
493         expected_html = u"""<html>
494   <head>
495     <title>Layout Test Results (time)</title>
496   </head>
497   <body>
498     <h2>Title (time)</h2>
499         <p><a href='test_path'>test_path</a><br />
500 </p>
501 </body></html>
502 """
503         html = runner._results_html(["test_path"], {}, "Title", override_time="time")
504         self.assertEqual(html, expected_html)
505
506     def test_shard_tests(self):
507         # Test that _shard_tests in run_webkit_tests.TestRunner really
508         # put the http tests first in the queue.
509         runner = TestRunnerWrapper(port=Mock(), options=Mock(),
510             printer=Mock(), message_broker=Mock())
511
512         test_list = [
513           "LayoutTests/websocket/tests/unicode.htm",
514           "LayoutTests/animations/keyframes.html",
515           "LayoutTests/http/tests/security/view-source-no-refresh.html",
516           "LayoutTests/websocket/tests/websocket-protocol-ignored.html",
517           "LayoutTests/fast/css/display-none-inline-style-change-crash.html",
518           "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
519           "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
520           "LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
521           "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
522         ]
523
524         expected_tests_to_http_lock = set([
525           'LayoutTests/websocket/tests/unicode.htm',
526           'LayoutTests/http/tests/security/view-source-no-refresh.html',
527           'LayoutTests/websocket/tests/websocket-protocol-ignored.html',
528           'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
529         ])
530
531         # FIXME: Ideally the HTTP tests don't have to all be in one shard.
532         single_thread_results = runner._shard_tests(test_list, False)
533         multi_thread_results = runner._shard_tests(test_list, True)
534
535         self.assertEqual("tests_to_http_lock", single_thread_results[0][0])
536         self.assertEqual(expected_tests_to_http_lock, set(single_thread_results[0][1]))
537         self.assertEqual("tests_to_http_lock", multi_thread_results[0][0])
538         self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1]))
539
540
541 class DryrunTest(unittest.TestCase):
542     # FIXME: it's hard to know which platforms are safe to test; the
543     # chromium platforms require a chromium checkout, and the mac platform
544     # requires fcntl, so it can't be tested on win32, etc. There is
545     # probably a better way of handling this.
546     def test_darwin(self):
547         if sys.platform != "darwin":
548             return
549
550         self.assertTrue(passing_run(['--platform', 'test']))
551         self.assertTrue(passing_run(['--platform', 'dryrun',
552                                      'fast/html']))
553         self.assertTrue(passing_run(['--platform', 'dryrun-mac',
554                                      'fast/html']))
555
556     def test_test(self):
557         self.assertTrue(passing_run(['--platform', 'dryrun-test',
558                                            '--pixel-tests']))
559
560
561 if __name__ == '__main__':
562     unittest.main()