2010-08-25 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import os
33 import optparse
34 import pdb
35 import sys
36 import unittest
37 import logging
38
39 from webkitpy.common import array_stream
40 from webkitpy.layout_tests import port
41 from webkitpy.layout_tests.layout_package import printing
42 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
43 from webkitpy.layout_tests.layout_package import test_expectations
44 from webkitpy.layout_tests.layout_package import test_failures
45 from webkitpy.layout_tests import run_webkit_tests
46
47
48 def get_options(args):
49     print_options = printing.print_options()
50     option_parser = optparse.OptionParser(option_list=print_options)
51     return option_parser.parse_args(args)
52
53
54 class TestUtilityFunctions(unittest.TestCase):
55     def test_configure_logging(self):
56         # FIXME: We need to figure out how to reset the basic logger.
57         # FIXME: If other testing classes call logging.basicConfig() then
58         # FIXME: these calls become no-ops and we can't control the
59         # FIXME: configuration to test things properly.
60         options, args = get_options([])
61         stream = array_stream.ArrayStream()
62         printing.configure_logging(options, stream)
63         logging.info("this should be logged")
64         # self.assertFalse(stream.empty())
65
66         stream.reset()
67         logging.debug("this should not be logged")
68         # self.assertTrue(stream.empty())
69
70         stream.reset()
71         options, args = get_options(['--verbose'])
72         printing.configure_logging(options, stream)
73         logging.debug("this should be logged")
74         # self.assertFalse(stream.empty())
75
76     def test_print_options(self):
77         options, args = get_options([])
78         self.assertTrue(options is not None)
79
80     def test_parse_print_options(self):
81         def test_switches(args, verbose, child_processes, is_fully_parallel,
82                           expected_switches_str):
83             options, args = get_options(args)
84             if expected_switches_str:
85                 expected_switches = set(expected_switches_str.split(','))
86             else:
87                 expected_switches = set()
88             switches = printing.parse_print_options(options.print_options,
89                                                     verbose,
90                                                     child_processes,
91                                                     is_fully_parallel)
92             self.assertEqual(expected_switches, switches)
93
94         # test that we default to the default set of switches
95         test_switches([], False, 1, False,
96                       printing.PRINT_DEFAULT)
97
98         # test that verbose defaults to everything
99         test_switches([], True, 1, False,
100                       printing.PRINT_EVERYTHING)
101
102         # test that --print default does what it's supposed to
103         test_switches(['--print', 'default'], False, 1, False,
104                       printing.PRINT_DEFAULT)
105
106         # test that --print nothing does what it's supposed to
107         test_switches(['--print', 'nothing'], False, 1, False,
108                       None)
109
110         # test that --print everything does what it's supposed to
111         test_switches(['--print', 'everything'], False, 1, False,
112                       printing.PRINT_EVERYTHING)
113
114         # this tests that '--print X' overrides '--verbose'
115         test_switches(['--print', 'actual'], True, 1, False,
116                       'actual')
117
118
119 class  Testprinter(unittest.TestCase):
120     def get_printer(self, args=None, single_threaded=False,
121                    is_fully_parallel=False):
122         printing_options = printing.print_options()
123         option_parser = optparse.OptionParser(option_list=printing_options)
124         options, args = option_parser.parse_args(args)
125         self._port = port.get('test', options)
126         nproc = 2
127         if single_threaded:
128             nproc = 1
129
130         regular_output = array_stream.ArrayStream()
131         buildbot_output = array_stream.ArrayStream()
132         printer = printing.Printer(self._port, options, regular_output,
133                                    buildbot_output, single_threaded,
134                                    is_fully_parallel)
135         return printer, regular_output, buildbot_output
136
137     def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
138         failures = []
139         if result_type == test_expectations.TIMEOUT:
140             failures = [test_failures.FailureTimeout()]
141         elif result_type == test_expectations.CRASH:
142             failures = [test_failures.FailureCrash()]
143         path = os.path.join(self._port.layout_tests_dir(), test)
144         return dump_render_tree_thread.TestResult(path, failures, run_time,
145                                                   total_time_for_all_diffs=0,
146                                                   time_for_diffs=0)
147
148     def get_result_summary(self, tests, expectations_str):
149         test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
150                       test in tests]
151         expectations = test_expectations.TestExpectations(
152             self._port, test_paths, expectations_str,
153             self._port.test_platform_name(), is_debug_mode=False,
154             is_lint_mode=False, tests_are_present=False)
155
156         rs = run_webkit_tests.ResultSummary(expectations, test_paths)
157         return test_paths, rs, expectations
158
159     def test_help_printer(self):
160         # Here and below we'll call the "regular" printer err and the
161         # buildbot printer out; this corresponds to how things run on the
162         # bots with stderr and stdout.
163         printer, err, out = self.get_printer()
164
165         # This routine should print something to stdout. testing what it is
166         # is kind of pointless.
167         printer.help_printing()
168         self.assertFalse(err.empty())
169         self.assertTrue(out.empty())
170
171     def do_switch_tests(self, method_name, switch, to_buildbot,
172                         message='hello', exp_err=None, exp_bot=None):
173         def do_helper(method_name, switch, message, exp_err, exp_bot):
174             printer, err, bot = self.get_printer(['--print', switch])
175             getattr(printer, method_name)(message)
176             self.assertEqual(err.get(), exp_err)
177             self.assertEqual(bot.get(), exp_bot)
178
179         if to_buildbot:
180             if exp_err is None:
181                 exp_err = []
182             if exp_bot is None:
183                 exp_bot = [message + "\n"]
184         else:
185             if exp_err is None:
186                 exp_err = [message + "\n"]
187             if exp_bot is None:
188                 exp_bot = []
189         do_helper(method_name, 'nothing', 'hello', [], [])
190         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
191         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
192
193     def test_print_actual(self):
194         # Actual results need to be logged to the buildbot's stream.
195         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
196
197     def test_print_actual_buildbot(self):
198         # FIXME: Test that the format of the actual results matches what the
199         # buildbot is expecting.
200         pass
201
202     def test_print_config(self):
203         self.do_switch_tests('print_config', 'config', to_buildbot=False)
204
205     def test_print_expected(self):
206         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
207
208     def test_print_timing(self):
209         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
210
211     def test_print_update(self):
212         # Note that there shouldn't be a carriage return here; updates()
213         # are meant to be overwritten.
214         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
215                              message='hello', exp_err=['hello'])
216
217     def test_print_one_line_summary(self):
218         printer, err, out = self.get_printer(['--print', 'nothing'])
219         printer.print_one_line_summary(1, 1, 0)
220         self.assertTrue(err.empty())
221
222         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
223         printer.print_one_line_summary(1, 1, 0)
224         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
225
226         printer, err, out = self.get_printer(['--print', 'everything'])
227         printer.print_one_line_summary(1, 1, 0)
228         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
229
230         err.reset()
231         printer.print_one_line_summary(2, 1, 1)
232         self.assertEquals(err.get(),
233                           ["1 test ran as expected, 1 didn't:\n", "\n"])
234
235         err.reset()
236         printer.print_one_line_summary(3, 2, 1)
237         self.assertEquals(err.get(),
238                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
239
240         err.reset()
241         printer.print_one_line_summary(3, 2, 0)
242         self.assertEquals(err.get(),
243                           ['\n', "2 tests ran as expected (1 didn't run).\n",
244                            '\n'])
245
246
247     def test_print_test_result(self):
248         # Note here that we don't use meaningful exp_str and got_str values;
249         # the actual contents of the string are treated opaquely by
250         # print_test_result() when tracing, and usually we don't want
251         # to test what exactly is printed, just that something
252         # was printed (or that nothing was printed).
253         #
254         # FIXME: this is actually some goofy layering; it would be nice
255         # we could refactor it so that the args weren't redundant. Maybe
256         # the TestResult should contain what was expected, and the
257         # strings could be derived from the TestResult?
258         printer, err, out = self.get_printer(['--print', 'nothing'])
259         result = self.get_result('passes/image.html')
260         printer.print_test_result(result, expected=False, exp_str='',
261                                   got_str='')
262         self.assertTrue(err.empty())
263
264         printer, err, out = self.get_printer(['--print', 'unexpected'])
265         printer.print_test_result(result, expected=True, exp_str='',
266                                   got_str='')
267         self.assertTrue(err.empty())
268         printer.print_test_result(result, expected=False, exp_str='',
269                                   got_str='')
270         self.assertEquals(err.get(),
271                           ['  passes/image.html -> unexpected pass\n'])
272
273         printer, err, out = self.get_printer(['--print', 'everything'])
274         printer.print_test_result(result, expected=True, exp_str='',
275                                   got_str='')
276         self.assertTrue(err.empty())
277
278         printer.print_test_result(result, expected=False, exp_str='',
279                                   got_str='')
280         self.assertEquals(err.get(),
281                           ['  passes/image.html -> unexpected pass\n'])
282
283         printer, err, out = self.get_printer(['--print', 'nothing'])
284         printer.print_test_result(result, expected=False, exp_str='',
285                                   got_str='')
286         self.assertTrue(err.empty())
287
288         printer, err, out = self.get_printer(['--print',
289                                               'trace-unexpected'])
290         printer.print_test_result(result, expected=True, exp_str='',
291                                   got_str='')
292         self.assertTrue(err.empty())
293
294         printer, err, out = self.get_printer(['--print',
295                                               'trace-unexpected'])
296         printer.print_test_result(result, expected=False, exp_str='',
297                                   got_str='')
298         self.assertFalse(err.empty())
299
300         printer, err, out = self.get_printer(['--print',
301                                               'trace-unexpected'])
302         result = self.get_result("passes/text.html")
303         printer.print_test_result(result, expected=False, exp_str='',
304                                   got_str='')
305         self.assertFalse(err.empty())
306
307         err.reset()
308         printer.print_test_result(result, expected=False, exp_str='',
309                                   got_str='')
310         self.assertFalse(err.empty())
311
312         printer, err, out = self.get_printer(['--print', 'trace-everything'])
313         printer.print_test_result(result, expected=True, exp_str='',
314                                   got_str='')
315         self.assertFalse(err.empty())
316
317         err.reset()
318         printer.print_test_result(result, expected=False, exp_str='',
319                                   got_str='')
320
321     def test_print_progress(self):
322         expectations = ''
323
324         # test that we print nothing
325         printer, err, out = self.get_printer(['--print', 'nothing'])
326         tests = ['passes/text.html', 'failures/expected/timeout.html',
327                  'failures/expected/crash.html']
328         paths, rs, exp = self.get_result_summary(tests, expectations)
329
330         printer.print_progress(rs, False, paths)
331         self.assertTrue(out.empty())
332         self.assertTrue(err.empty())
333
334         printer.print_progress(rs, True, paths)
335         self.assertTrue(out.empty())
336         self.assertTrue(err.empty())
337
338         # test regular functionality
339         printer, err, out = self.get_printer(['--print',
340                                               'one-line-progress'])
341         printer.print_progress(rs, False, paths)
342         self.assertTrue(out.empty())
343         self.assertFalse(err.empty())
344
345         err.reset()
346         out.reset()
347         printer.print_progress(rs, True, paths)
348         self.assertFalse(err.empty())
349         self.assertTrue(out.empty())
350
351     def test_print_progress__detailed(self):
352         tests = ['passes/text.html', 'failures/expected/timeout.html',
353                  'failures/expected/crash.html']
354         expectations = 'failures/expected/timeout.html = TIMEOUT'
355
356         # first, test that it is disabled properly
357         # should still print one-line-progress
358         printer, err, out = self.get_printer(
359             ['--print', 'detailed-progress'], single_threaded=False)
360         paths, rs, exp = self.get_result_summary(tests, expectations)
361         printer.print_progress(rs, False, paths)
362         self.assertFalse(err.empty())
363         self.assertTrue(out.empty())
364
365         # now test the enabled paths
366         printer, err, out = self.get_printer(
367             ['--print', 'detailed-progress'], single_threaded=True)
368         paths, rs, exp = self.get_result_summary(tests, expectations)
369         printer.print_progress(rs, False, paths)
370         self.assertFalse(err.empty())
371         self.assertTrue(out.empty())
372
373         err.reset()
374         out.reset()
375         printer.print_progress(rs, True, paths)
376         self.assertFalse(err.empty())
377         self.assertTrue(out.empty())
378
379         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
380         rs.add(self.get_result('failures/expected/timeout.html'), True)
381         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
382         err.reset()
383         out.reset()
384         printer.print_progress(rs, False, paths)
385         self.assertFalse(err.empty())
386         self.assertTrue(out.empty())
387
388         # We only clear the meter when retrying w/ detailed-progress.
389         err.reset()
390         out.reset()
391         printer.print_progress(rs, True, paths)
392         self.assertFalse(err.empty())
393         self.assertTrue(out.empty())
394
395         printer, err, out = self.get_printer(
396             ['--print', 'detailed-progress,unexpected'], single_threaded=True)
397         paths, rs, exp = self.get_result_summary(tests, expectations)
398         printer.print_progress(rs, False, paths)
399         self.assertFalse(err.empty())
400         self.assertTrue(out.empty())
401
402         err.reset()
403         out.reset()
404         printer.print_progress(rs, True, paths)
405         self.assertFalse(err.empty())
406         self.assertTrue(out.empty())
407
408         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
409         rs.add(self.get_result('failures/expected/timeout.html'), True)
410         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
411         err.reset()
412         out.reset()
413         printer.print_progress(rs, False, paths)
414         self.assertFalse(err.empty())
415         self.assertTrue(out.empty())
416
417         # We only clear the meter when retrying w/ detailed-progress.
418         err.reset()
419         out.reset()
420         printer.print_progress(rs, True, paths)
421         self.assertFalse(err.empty())
422         self.assertTrue(out.empty())
423
424     def test_write(self):
425         printer, err, out = self.get_printer(['--print', 'nothing'])
426         printer.write("foo")
427         self.assertTrue(err.empty())
428
429         printer, err, out = self.get_printer(['--print', 'misc'])
430         printer.write("foo")
431         self.assertFalse(err.empty())
432         err.reset()
433         printer.write("foo", "config")
434         self.assertTrue(err.empty())
435
436         printer, err, out = self.get_printer(['--print', 'everything'])
437         printer.write("foo")
438         self.assertFalse(err.empty())
439         err.reset()
440         printer.write("foo", "config")
441         self.assertFalse(err.empty())
442
443         # FIXME: this should be logged somewhere, but it actually
444         # disappears into the ether in the logging subsystem.
445         printer, err, out = self.get_printer(['--verbose'])
446         printer.write("foo")
447         self.assertTrue(err.empty())
448         self.assertTrue(out.empty())
449
450     def test_print_unexpected_results(self):
451         # This routine is the only one that prints stuff that the bots
452         # care about.
453         #
454         # FIXME: there's some weird layering going on here. It seems
455         # like we shouldn't be both using an expectations string and
456         # having to specify whether or not the result was expected.
457         # This whole set of tests should probably be rewritten.
458         #
459         # FIXME: Plus, the fact that we're having to call into
460         # run_webkit_tests is clearly a layering inversion.
461         def get_unexpected_results(expected, passing, flaky):
462             """Return an unexpected results summary matching the input description.
463
464             There are a lot of different combinations of test results that
465             can be tested; this routine produces various combinations based
466             on the values of the input flags.
467
468             Args
469                 expected: whether the tests ran as expected
470                 passing: whether the tests should all pass
471                 flaky: whether the tests should be flaky (if False, they
472                     produce the same results on both runs; if True, they
473                     all pass on the second run).
474
475             """
476             paths, rs, exp = self.get_result_summary(tests, expectations)
477             if expected:
478                 rs.add(self.get_result('passes/text.html', test_expectations.PASS),
479                        expected)
480                 rs.add(self.get_result('failures/expected/timeout.html',
481                        test_expectations.TIMEOUT), expected)
482                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
483                    expected)
484             elif passing:
485                 rs.add(self.get_result('passes/text.html'), expected)
486                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
487                 rs.add(self.get_result('failures/expected/crash.html'), expected)
488             else:
489                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
490                        expected)
491                 rs.add(self.get_result('failures/expected/timeout.html',
492                        test_expectations.CRASH), expected)
493                 rs.add(self.get_result('failures/expected/crash.html',
494                                   test_expectations.TIMEOUT),
495                    expected)
496             retry = rs
497             if flaky:
498                 paths, retry, exp = self.get_result_summary(tests,
499                                                 expectations)
500                 retry.add(self.get_result('passes/text.html'), True)
501                 retry.add(self.get_result('failures/expected/timeout.html'), True)
502                 retry.add(self.get_result('failures/expected/crash.html'), True)
503             unexpected_results = run_webkit_tests.summarize_unexpected_results(
504                 self._port, exp, rs, retry)
505             return unexpected_results
506
507         tests = ['passes/text.html', 'failures/expected/timeout.html',
508                  'failures/expected/crash.html']
509         expectations = ''
510
511         printer, err, out = self.get_printer(['--print', 'nothing'])
512         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
513         printer.print_unexpected_results(ur)
514         self.assertTrue(err.empty())
515         self.assertTrue(out.empty())
516
517         printer, err, out = self.get_printer(['--print',
518                                               'unexpected-results'])
519
520         # test everything running as expected
521         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
522         printer.print_unexpected_results(ur)
523         self.assertTrue(err.empty())
524         self.assertTrue(out.empty())
525
526         # test failures
527         err.reset()
528         out.reset()
529         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
530         printer.print_unexpected_results(ur)
531         self.assertTrue(err.empty())
532         self.assertFalse(out.empty())
533
534         # test unexpected flaky results
535         err.reset()
536         out.reset()
537         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
538         printer.print_unexpected_results(ur)
539         self.assertTrue(err.empty())
540         self.assertFalse(out.empty())
541
542         # test unexpected passes
543         err.reset()
544         out.reset()
545         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
546         printer.print_unexpected_results(ur)
547         self.assertTrue(err.empty())
548         self.assertFalse(out.empty())
549
550         err.reset()
551         out.reset()
552         printer, err, out = self.get_printer(['--print', 'everything'])
553         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
554         printer.print_unexpected_results(ur)
555         self.assertTrue(err.empty())
556         self.assertFalse(out.empty())
557
558         expectations = """
559 failures/expected/crash.html = CRASH
560 failures/expected/timeout.html = TIMEOUT
561 """
562         err.reset()
563         out.reset()
564         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
565         printer.print_unexpected_results(ur)
566         self.assertTrue(err.empty())
567         self.assertFalse(out.empty())
568
569         err.reset()
570         out.reset()
571         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
572         printer.print_unexpected_results(ur)
573         self.assertTrue(err.empty())
574         self.assertFalse(out.empty())
575
576         # Test handling of --verbose as well.
577         err.reset()
578         out.reset()
579         printer, err, out = self.get_printer(['--verbose'])
580         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
581         printer.print_unexpected_results(ur)
582         self.assertTrue(err.empty())
583         self.assertFalse(out.empty())
584
585     def test_print_unexpected_results_buildbot(self):
586         # FIXME: Test that print_unexpected_results() produces the printer the
587         # buildbot is expecting.
588         pass
589
590 if __name__ == '__main__':
591     unittest.main()