2010-12-22 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import os
33 import optparse
34 import pdb
35 import sys
36 import unittest
37 import logging
38
39 from webkitpy.common import array_stream
40 from webkitpy.common.system import logtesting
41 from webkitpy.layout_tests import port
42
43 from webkitpy.layout_tests.layout_package import printing
44 from webkitpy.layout_tests.layout_package import result_summary
45 from webkitpy.layout_tests.layout_package import test_expectations
46 from webkitpy.layout_tests.layout_package import test_failures
47 from webkitpy.layout_tests.layout_package import test_results
48 from webkitpy.layout_tests.layout_package import test_runner
49
50
51 def get_options(args):
52     print_options = printing.print_options()
53     option_parser = optparse.OptionParser(option_list=print_options)
54     return option_parser.parse_args(args)
55
56
57 class TestUtilityFunctions(unittest.TestCase):
58     def test_configure_logging(self):
59         options, args = get_options([])
60         stream = array_stream.ArrayStream()
61         handler = printing._configure_logging(stream, options.verbose)
62         logging.info("this should be logged")
63         self.assertFalse(stream.empty())
64
65         stream.reset()
66         logging.debug("this should not be logged")
67         self.assertTrue(stream.empty())
68
69         printing._restore_logging(handler)
70
71         stream.reset()
72         options, args = get_options(['--verbose'])
73         handler = printing._configure_logging(stream, options.verbose)
74         logging.debug("this should be logged")
75         self.assertFalse(stream.empty())
76         printing._restore_logging(handler)
77
78     def test_print_options(self):
79         options, args = get_options([])
80         self.assertTrue(options is not None)
81
82     def test_parse_print_options(self):
83         def test_switches(args, expected_switches_str,
84                           verbose=False, child_processes=1,
85                           is_fully_parallel=False):
86             options, args = get_options(args)
87             if expected_switches_str:
88                 expected_switches = set(expected_switches_str.split(','))
89             else:
90                 expected_switches = set()
91             switches = printing.parse_print_options(options.print_options,
92                                                     verbose,
93                                                     child_processes,
94                                                     is_fully_parallel)
95             self.assertEqual(expected_switches, switches)
96
97         # test that we default to the default set of switches
98         test_switches([], printing.PRINT_DEFAULT)
99
100         # test that verbose defaults to everything
101         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
102
103         # test that --print default does what it's supposed to
104         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
105
106         # test that --print nothing does what it's supposed to
107         test_switches(['--print', 'nothing'], None)
108
109         # test that --print everything does what it's supposed to
110         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
111
112         # this tests that '--print X' overrides '--verbose'
113         test_switches(['--print', 'actual'], 'actual', verbose=True)
114
115
116
117 class  Testprinter(unittest.TestCase):
118     def get_printer(self, args=None, single_threaded=False,
119                    is_fully_parallel=False):
120         printing_options = printing.print_options()
121         option_parser = optparse.OptionParser(option_list=printing_options)
122         options, args = option_parser.parse_args(args)
123         self._port = port.get('test', options)
124         nproc = 2
125         if single_threaded:
126             nproc = 1
127
128         regular_output = array_stream.ArrayStream()
129         buildbot_output = array_stream.ArrayStream()
130         printer = printing.Printer(self._port, options, regular_output,
131                                    buildbot_output, single_threaded,
132                                    is_fully_parallel)
133         return printer, regular_output, buildbot_output
134
135     def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
136         failures = []
137         if result_type == test_expectations.TIMEOUT:
138             failures = [test_failures.FailureTimeout()]
139         elif result_type == test_expectations.CRASH:
140             failures = [test_failures.FailureCrash()]
141         path = os.path.join(self._port.layout_tests_dir(), test)
142         return test_results.TestResult(path, failures, run_time,
143                                        total_time_for_all_diffs=0,
144                                        time_for_diffs=0)
145
146     def get_result_summary(self, tests, expectations_str):
147         test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
148                       test in tests]
149         expectations = test_expectations.TestExpectations(
150             self._port, test_paths, expectations_str,
151             self._port.test_platform_name(), is_debug_mode=False,
152             is_lint_mode=False)
153
154         rs = result_summary.ResultSummary(expectations, test_paths)
155         return test_paths, rs, expectations
156
157     def test_help_printer(self):
158         # Here and below we'll call the "regular" printer err and the
159         # buildbot printer out; this corresponds to how things run on the
160         # bots with stderr and stdout.
161         printer, err, out = self.get_printer()
162
163         # This routine should print something to stdout. testing what it is
164         # is kind of pointless.
165         printer.help_printing()
166         self.assertFalse(err.empty())
167         self.assertTrue(out.empty())
168
169     def do_switch_tests(self, method_name, switch, to_buildbot,
170                         message='hello', exp_err=None, exp_bot=None):
171         def do_helper(method_name, switch, message, exp_err, exp_bot):
172             printer, err, bot = self.get_printer(['--print', switch])
173             getattr(printer, method_name)(message)
174             self.assertEqual(err.get(), exp_err)
175             self.assertEqual(bot.get(), exp_bot)
176
177         if to_buildbot:
178             if exp_err is None:
179                 exp_err = []
180             if exp_bot is None:
181                 exp_bot = [message + "\n"]
182         else:
183             if exp_err is None:
184                 exp_err = [message + "\n"]
185             if exp_bot is None:
186                 exp_bot = []
187         do_helper(method_name, 'nothing', 'hello', [], [])
188         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
189         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
190
191     def test_configure_and_cleanup(self):
192         # This test verifies that calling cleanup repeatedly and deleting
193         # the object is safe.
194         printer, err, out = self.get_printer(['--print', 'everything'])
195         printer.cleanup()
196         printer.cleanup()
197         printer = None
198
199     def test_print_actual(self):
200         # Actual results need to be logged to the buildbot's stream.
201         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
202
203     def test_print_actual_buildbot(self):
204         # FIXME: Test that the format of the actual results matches what the
205         # buildbot is expecting.
206         pass
207
208     def test_print_config(self):
209         self.do_switch_tests('print_config', 'config', to_buildbot=False)
210
211     def test_print_expected(self):
212         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
213
214     def test_print_timing(self):
215         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
216
217     def test_print_update(self):
218         # Note that there shouldn't be a carriage return here; updates()
219         # are meant to be overwritten.
220         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
221                              message='hello', exp_err=['hello'])
222
223     def test_print_one_line_summary(self):
224         printer, err, out = self.get_printer(['--print', 'nothing'])
225         printer.print_one_line_summary(1, 1, 0)
226         self.assertTrue(err.empty())
227
228         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
229         printer.print_one_line_summary(1, 1, 0)
230         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
231
232         printer, err, out = self.get_printer(['--print', 'everything'])
233         printer.print_one_line_summary(1, 1, 0)
234         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
235
236         err.reset()
237         printer.print_one_line_summary(2, 1, 1)
238         self.assertEquals(err.get(),
239                           ["1 test ran as expected, 1 didn't:\n", "\n"])
240
241         err.reset()
242         printer.print_one_line_summary(3, 2, 1)
243         self.assertEquals(err.get(),
244                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
245
246         err.reset()
247         printer.print_one_line_summary(3, 2, 0)
248         self.assertEquals(err.get(),
249                           ['\n', "2 tests ran as expected (1 didn't run).\n",
250                            '\n'])
251
252
253     def test_print_test_result(self):
254         # Note here that we don't use meaningful exp_str and got_str values;
255         # the actual contents of the string are treated opaquely by
256         # print_test_result() when tracing, and usually we don't want
257         # to test what exactly is printed, just that something
258         # was printed (or that nothing was printed).
259         #
260         # FIXME: this is actually some goofy layering; it would be nice
261         # we could refactor it so that the args weren't redundant. Maybe
262         # the TestResult should contain what was expected, and the
263         # strings could be derived from the TestResult?
264         printer, err, out = self.get_printer(['--print', 'nothing'])
265         result = self.get_result('passes/image.html')
266         printer.print_test_result(result, expected=False, exp_str='',
267                                   got_str='')
268         self.assertTrue(err.empty())
269
270         printer, err, out = self.get_printer(['--print', 'unexpected'])
271         printer.print_test_result(result, expected=True, exp_str='',
272                                   got_str='')
273         self.assertTrue(err.empty())
274         printer.print_test_result(result, expected=False, exp_str='',
275                                   got_str='')
276         self.assertEquals(err.get(),
277                           ['  passes/image.html -> unexpected pass\n'])
278
279         printer, err, out = self.get_printer(['--print', 'everything'])
280         printer.print_test_result(result, expected=True, exp_str='',
281                                   got_str='')
282         self.assertTrue(err.empty())
283
284         printer.print_test_result(result, expected=False, exp_str='',
285                                   got_str='')
286         self.assertEquals(err.get(),
287                           ['  passes/image.html -> unexpected pass\n'])
288
289         printer, err, out = self.get_printer(['--print', 'nothing'])
290         printer.print_test_result(result, expected=False, exp_str='',
291                                   got_str='')
292         self.assertTrue(err.empty())
293
294         printer, err, out = self.get_printer(['--print',
295                                               'trace-unexpected'])
296         printer.print_test_result(result, expected=True, exp_str='',
297                                   got_str='')
298         self.assertTrue(err.empty())
299
300         printer, err, out = self.get_printer(['--print',
301                                               'trace-unexpected'])
302         printer.print_test_result(result, expected=False, exp_str='',
303                                   got_str='')
304         self.assertFalse(err.empty())
305
306         printer, err, out = self.get_printer(['--print',
307                                               'trace-unexpected'])
308         result = self.get_result("passes/text.html")
309         printer.print_test_result(result, expected=False, exp_str='',
310                                   got_str='')
311         self.assertFalse(err.empty())
312
313         err.reset()
314         printer.print_test_result(result, expected=False, exp_str='',
315                                   got_str='')
316         self.assertFalse(err.empty())
317
318         printer, err, out = self.get_printer(['--print', 'trace-everything'])
319         result = self.get_result('passes/image.html')
320         printer.print_test_result(result, expected=True, exp_str='',
321                                   got_str='')
322         result = self.get_result('failures/expected/missing_text.html')
323         printer.print_test_result(result, expected=True, exp_str='',
324                                   got_str='')
325         result = self.get_result('failures/expected/missing_check.html')
326         printer.print_test_result(result, expected=True, exp_str='',
327                                   got_str='')
328         result = self.get_result('failures/expected/missing_image.html')
329         printer.print_test_result(result, expected=True, exp_str='',
330                                   got_str='')
331         self.assertFalse(err.empty())
332
333         err.reset()
334         printer.print_test_result(result, expected=False, exp_str='',
335                                   got_str='')
336
337     def test_print_progress(self):
338         expectations = ''
339
340         # test that we print nothing
341         printer, err, out = self.get_printer(['--print', 'nothing'])
342         tests = ['passes/text.html', 'failures/expected/timeout.html',
343                  'failures/expected/crash.html']
344         paths, rs, exp = self.get_result_summary(tests, expectations)
345
346         printer.print_progress(rs, False, paths)
347         self.assertTrue(out.empty())
348         self.assertTrue(err.empty())
349
350         printer.print_progress(rs, True, paths)
351         self.assertTrue(out.empty())
352         self.assertTrue(err.empty())
353
354         # test regular functionality
355         printer, err, out = self.get_printer(['--print',
356                                               'one-line-progress'])
357         printer.print_progress(rs, False, paths)
358         self.assertTrue(out.empty())
359         self.assertFalse(err.empty())
360
361         err.reset()
362         out.reset()
363         printer.print_progress(rs, True, paths)
364         self.assertFalse(err.empty())
365         self.assertTrue(out.empty())
366
367     def test_print_progress__detailed(self):
368         tests = ['passes/text.html', 'failures/expected/timeout.html',
369                  'failures/expected/crash.html']
370         expectations = 'failures/expected/timeout.html = TIMEOUT'
371
372         # first, test that it is disabled properly
373         # should still print one-line-progress
374         printer, err, out = self.get_printer(
375             ['--print', 'detailed-progress'], single_threaded=False)
376         paths, rs, exp = self.get_result_summary(tests, expectations)
377         printer.print_progress(rs, False, paths)
378         self.assertFalse(err.empty())
379         self.assertTrue(out.empty())
380
381         # now test the enabled paths
382         printer, err, out = self.get_printer(
383             ['--print', 'detailed-progress'], single_threaded=True)
384         paths, rs, exp = self.get_result_summary(tests, expectations)
385         printer.print_progress(rs, False, paths)
386         self.assertFalse(err.empty())
387         self.assertTrue(out.empty())
388
389         err.reset()
390         out.reset()
391         printer.print_progress(rs, True, paths)
392         self.assertFalse(err.empty())
393         self.assertTrue(out.empty())
394
395         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
396         rs.add(self.get_result('failures/expected/timeout.html'), True)
397         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
398         err.reset()
399         out.reset()
400         printer.print_progress(rs, False, paths)
401         self.assertFalse(err.empty())
402         self.assertTrue(out.empty())
403
404         # We only clear the meter when retrying w/ detailed-progress.
405         err.reset()
406         out.reset()
407         printer.print_progress(rs, True, paths)
408         self.assertFalse(err.empty())
409         self.assertTrue(out.empty())
410
411         printer, err, out = self.get_printer(
412             ['--print', 'detailed-progress,unexpected'], single_threaded=True)
413         paths, rs, exp = self.get_result_summary(tests, expectations)
414         printer.print_progress(rs, False, paths)
415         self.assertFalse(err.empty())
416         self.assertTrue(out.empty())
417
418         err.reset()
419         out.reset()
420         printer.print_progress(rs, True, paths)
421         self.assertFalse(err.empty())
422         self.assertTrue(out.empty())
423
424         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
425         rs.add(self.get_result('failures/expected/timeout.html'), True)
426         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
427         err.reset()
428         out.reset()
429         printer.print_progress(rs, False, paths)
430         self.assertFalse(err.empty())
431         self.assertTrue(out.empty())
432
433         # We only clear the meter when retrying w/ detailed-progress.
434         err.reset()
435         out.reset()
436         printer.print_progress(rs, True, paths)
437         self.assertFalse(err.empty())
438         self.assertTrue(out.empty())
439
440     def test_write_nothing(self):
441         printer, err, out = self.get_printer(['--print', 'nothing'])
442         printer.write("foo")
443         self.assertTrue(err.empty())
444
445     def test_write_misc(self):
446         printer, err, out = self.get_printer(['--print', 'misc'])
447         printer.write("foo")
448         self.assertFalse(err.empty())
449         err.reset()
450         printer.write("foo", "config")
451         self.assertTrue(err.empty())
452
453     def test_write_everything(self):
454         printer, err, out = self.get_printer(['--print', 'everything'])
455         printer.write("foo")
456         self.assertFalse(err.empty())
457         err.reset()
458         printer.write("foo", "config")
459         self.assertFalse(err.empty())
460
461     def test_write_verbose(self):
462         printer, err, out = self.get_printer(['--verbose'])
463         printer.write("foo")
464         self.assertTrue(not err.empty() and "foo" in err.get()[0])
465         self.assertTrue(out.empty())
466
467     def test_print_unexpected_results(self):
468         # This routine is the only one that prints stuff that the bots
469         # care about.
470         #
471         # FIXME: there's some weird layering going on here. It seems
472         # like we shouldn't be both using an expectations string and
473         # having to specify whether or not the result was expected.
474         # This whole set of tests should probably be rewritten.
475         #
476         # FIXME: Plus, the fact that we're having to call into
477         # run_webkit_tests is clearly a layering inversion.
478         def get_unexpected_results(expected, passing, flaky):
479             """Return an unexpected results summary matching the input description.
480
481             There are a lot of different combinations of test results that
482             can be tested; this routine produces various combinations based
483             on the values of the input flags.
484
485             Args
486                 expected: whether the tests ran as expected
487                 passing: whether the tests should all pass
488                 flaky: whether the tests should be flaky (if False, they
489                     produce the same results on both runs; if True, they
490                     all pass on the second run).
491
492             """
493             paths, rs, exp = self.get_result_summary(tests, expectations)
494             if expected:
495                 rs.add(self.get_result('passes/text.html', test_expectations.PASS),
496                        expected)
497                 rs.add(self.get_result('failures/expected/timeout.html',
498                        test_expectations.TIMEOUT), expected)
499                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
500                    expected)
501             elif passing:
502                 rs.add(self.get_result('passes/text.html'), expected)
503                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
504                 rs.add(self.get_result('failures/expected/crash.html'), expected)
505             else:
506                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
507                        expected)
508                 rs.add(self.get_result('failures/expected/timeout.html',
509                        test_expectations.CRASH), expected)
510                 rs.add(self.get_result('failures/expected/crash.html',
511                                   test_expectations.TIMEOUT),
512                    expected)
513             retry = rs
514             if flaky:
515                 paths, retry, exp = self.get_result_summary(tests,
516                                                 expectations)
517                 retry.add(self.get_result('passes/text.html'), True)
518                 retry.add(self.get_result('failures/expected/timeout.html'), True)
519                 retry.add(self.get_result('failures/expected/crash.html'), True)
520             unexpected_results = test_runner.summarize_unexpected_results(
521                 self._port, exp, rs, retry)
522             return unexpected_results
523
524         tests = ['passes/text.html', 'failures/expected/timeout.html',
525                  'failures/expected/crash.html']
526         expectations = ''
527
528         printer, err, out = self.get_printer(['--print', 'nothing'])
529         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
530         printer.print_unexpected_results(ur)
531         self.assertTrue(err.empty())
532         self.assertTrue(out.empty())
533
534         printer, err, out = self.get_printer(['--print',
535                                               'unexpected-results'])
536
537         # test everything running as expected
538         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
539         printer.print_unexpected_results(ur)
540         self.assertTrue(err.empty())
541         self.assertTrue(out.empty())
542
543         # test failures
544         err.reset()
545         out.reset()
546         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
547         printer.print_unexpected_results(ur)
548         self.assertTrue(err.empty())
549         self.assertFalse(out.empty())
550
551         # test unexpected flaky results
552         err.reset()
553         out.reset()
554         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
555         printer.print_unexpected_results(ur)
556         self.assertTrue(err.empty())
557         self.assertFalse(out.empty())
558
559         # test unexpected passes
560         err.reset()
561         out.reset()
562         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
563         printer.print_unexpected_results(ur)
564         self.assertTrue(err.empty())
565         self.assertFalse(out.empty())
566
567         err.reset()
568         out.reset()
569         printer, err, out = self.get_printer(['--print', 'everything'])
570         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
571         printer.print_unexpected_results(ur)
572         self.assertTrue(err.empty())
573         self.assertFalse(out.empty())
574
575         expectations = """
576 failures/expected/crash.html = CRASH
577 failures/expected/timeout.html = TIMEOUT
578 """
579         err.reset()
580         out.reset()
581         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
582         printer.print_unexpected_results(ur)
583         self.assertTrue(err.empty())
584         self.assertFalse(out.empty())
585
586         err.reset()
587         out.reset()
588         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
589         printer.print_unexpected_results(ur)
590         self.assertTrue(err.empty())
591         self.assertFalse(out.empty())
592
593         # Test handling of --verbose as well.
594         err.reset()
595         out.reset()
596         printer, err, out = self.get_printer(['--verbose'])
597         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
598         printer.print_unexpected_results(ur)
599         self.assertTrue(err.empty())
600         self.assertFalse(out.empty())
601
602     def test_print_unexpected_results_buildbot(self):
603         # FIXME: Test that print_unexpected_results() produces the printer the
604         # buildbot is expecting.
605         pass
606
607 if __name__ == '__main__':
608     unittest.main()