Rename WebKitTools to Tools
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import os
33 import optparse
34 import pdb
35 import sys
36 import unittest
37 import logging
38
39 from webkitpy.common import array_stream
40 from webkitpy.common.system import logtesting
41 from webkitpy.layout_tests import port
42 from webkitpy.layout_tests.layout_package import printing
43 from webkitpy.layout_tests.layout_package import test_results
44 from webkitpy.layout_tests.layout_package import test_expectations
45 from webkitpy.layout_tests.layout_package import test_failures
46 from webkitpy.layout_tests import run_webkit_tests
47
48
49 def get_options(args):
50     print_options = printing.print_options()
51     option_parser = optparse.OptionParser(option_list=print_options)
52     return option_parser.parse_args(args)
53
54
55 class TestUtilityFunctions(unittest.TestCase):
56     def test_configure_logging(self):
57         options, args = get_options([])
58         stream = array_stream.ArrayStream()
59         handler = printing._configure_logging(stream, options.verbose)
60         logging.info("this should be logged")
61         self.assertFalse(stream.empty())
62
63         stream.reset()
64         logging.debug("this should not be logged")
65         self.assertTrue(stream.empty())
66
67         printing._restore_logging(handler)
68
69         stream.reset()
70         options, args = get_options(['--verbose'])
71         handler = printing._configure_logging(stream, options.verbose)
72         logging.debug("this should be logged")
73         self.assertFalse(stream.empty())
74         printing._restore_logging(handler)
75
76     def test_print_options(self):
77         options, args = get_options([])
78         self.assertTrue(options is not None)
79
80     def test_parse_print_options(self):
81         def test_switches(args, expected_switches_str,
82                           verbose=False, child_processes=1,
83                           is_fully_parallel=False):
84             options, args = get_options(args)
85             if expected_switches_str:
86                 expected_switches = set(expected_switches_str.split(','))
87             else:
88                 expected_switches = set()
89             switches = printing.parse_print_options(options.print_options,
90                                                     verbose,
91                                                     child_processes,
92                                                     is_fully_parallel)
93             self.assertEqual(expected_switches, switches)
94
95         # test that we default to the default set of switches
96         test_switches([], printing.PRINT_DEFAULT)
97
98         # test that verbose defaults to everything
99         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
100
101         # test that --print default does what it's supposed to
102         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
103
104         # test that --print nothing does what it's supposed to
105         test_switches(['--print', 'nothing'], None)
106
107         # test that --print everything does what it's supposed to
108         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
109
110         # this tests that '--print X' overrides '--verbose'
111         test_switches(['--print', 'actual'], 'actual', verbose=True)
112
113
114
115 class  Testprinter(unittest.TestCase):
116     def get_printer(self, args=None, single_threaded=False,
117                    is_fully_parallel=False):
118         printing_options = printing.print_options()
119         option_parser = optparse.OptionParser(option_list=printing_options)
120         options, args = option_parser.parse_args(args)
121         self._port = port.get('test', options)
122         nproc = 2
123         if single_threaded:
124             nproc = 1
125
126         regular_output = array_stream.ArrayStream()
127         buildbot_output = array_stream.ArrayStream()
128         printer = printing.Printer(self._port, options, regular_output,
129                                    buildbot_output, single_threaded,
130                                    is_fully_parallel)
131         return printer, regular_output, buildbot_output
132
133     def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
134         failures = []
135         if result_type == test_expectations.TIMEOUT:
136             failures = [test_failures.FailureTimeout()]
137         elif result_type == test_expectations.CRASH:
138             failures = [test_failures.FailureCrash()]
139         path = os.path.join(self._port.layout_tests_dir(), test)
140         return test_results.TestResult(path, failures, run_time,
141                                        total_time_for_all_diffs=0,
142                                        time_for_diffs=0)
143
144     def get_result_summary(self, tests, expectations_str):
145         test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
146                       test in tests]
147         expectations = test_expectations.TestExpectations(
148             self._port, test_paths, expectations_str,
149             self._port.test_platform_name(), is_debug_mode=False,
150             is_lint_mode=False)
151
152         rs = run_webkit_tests.ResultSummary(expectations, test_paths)
153         return test_paths, rs, expectations
154
155     def test_help_printer(self):
156         # Here and below we'll call the "regular" printer err and the
157         # buildbot printer out; this corresponds to how things run on the
158         # bots with stderr and stdout.
159         printer, err, out = self.get_printer()
160
161         # This routine should print something to stdout. testing what it is
162         # is kind of pointless.
163         printer.help_printing()
164         self.assertFalse(err.empty())
165         self.assertTrue(out.empty())
166
167     def do_switch_tests(self, method_name, switch, to_buildbot,
168                         message='hello', exp_err=None, exp_bot=None):
169         def do_helper(method_name, switch, message, exp_err, exp_bot):
170             printer, err, bot = self.get_printer(['--print', switch])
171             getattr(printer, method_name)(message)
172             self.assertEqual(err.get(), exp_err)
173             self.assertEqual(bot.get(), exp_bot)
174
175         if to_buildbot:
176             if exp_err is None:
177                 exp_err = []
178             if exp_bot is None:
179                 exp_bot = [message + "\n"]
180         else:
181             if exp_err is None:
182                 exp_err = [message + "\n"]
183             if exp_bot is None:
184                 exp_bot = []
185         do_helper(method_name, 'nothing', 'hello', [], [])
186         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
187         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
188
189     def test_configure_and_cleanup(self):
190         # This test verifies that calling cleanup repeatedly and deleting
191         # the object is safe.
192         printer, err, out = self.get_printer(['--print', 'everything'])
193         printer.cleanup()
194         printer.cleanup()
195         printer = None
196
197     def test_print_actual(self):
198         # Actual results need to be logged to the buildbot's stream.
199         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
200
201     def test_print_actual_buildbot(self):
202         # FIXME: Test that the format of the actual results matches what the
203         # buildbot is expecting.
204         pass
205
206     def test_print_config(self):
207         self.do_switch_tests('print_config', 'config', to_buildbot=False)
208
209     def test_print_expected(self):
210         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
211
212     def test_print_timing(self):
213         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
214
215     def test_print_update(self):
216         # Note that there shouldn't be a carriage return here; updates()
217         # are meant to be overwritten.
218         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
219                              message='hello', exp_err=['hello'])
220
221     def test_print_one_line_summary(self):
222         printer, err, out = self.get_printer(['--print', 'nothing'])
223         printer.print_one_line_summary(1, 1, 0)
224         self.assertTrue(err.empty())
225
226         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
227         printer.print_one_line_summary(1, 1, 0)
228         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
229
230         printer, err, out = self.get_printer(['--print', 'everything'])
231         printer.print_one_line_summary(1, 1, 0)
232         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
233
234         err.reset()
235         printer.print_one_line_summary(2, 1, 1)
236         self.assertEquals(err.get(),
237                           ["1 test ran as expected, 1 didn't:\n", "\n"])
238
239         err.reset()
240         printer.print_one_line_summary(3, 2, 1)
241         self.assertEquals(err.get(),
242                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
243
244         err.reset()
245         printer.print_one_line_summary(3, 2, 0)
246         self.assertEquals(err.get(),
247                           ['\n', "2 tests ran as expected (1 didn't run).\n",
248                            '\n'])
249
250
251     def test_print_test_result(self):
252         # Note here that we don't use meaningful exp_str and got_str values;
253         # the actual contents of the string are treated opaquely by
254         # print_test_result() when tracing, and usually we don't want
255         # to test what exactly is printed, just that something
256         # was printed (or that nothing was printed).
257         #
258         # FIXME: this is actually some goofy layering; it would be nice
259         # we could refactor it so that the args weren't redundant. Maybe
260         # the TestResult should contain what was expected, and the
261         # strings could be derived from the TestResult?
262         printer, err, out = self.get_printer(['--print', 'nothing'])
263         result = self.get_result('passes/image.html')
264         printer.print_test_result(result, expected=False, exp_str='',
265                                   got_str='')
266         self.assertTrue(err.empty())
267
268         printer, err, out = self.get_printer(['--print', 'unexpected'])
269         printer.print_test_result(result, expected=True, exp_str='',
270                                   got_str='')
271         self.assertTrue(err.empty())
272         printer.print_test_result(result, expected=False, exp_str='',
273                                   got_str='')
274         self.assertEquals(err.get(),
275                           ['  passes/image.html -> unexpected pass\n'])
276
277         printer, err, out = self.get_printer(['--print', 'everything'])
278         printer.print_test_result(result, expected=True, exp_str='',
279                                   got_str='')
280         self.assertTrue(err.empty())
281
282         printer.print_test_result(result, expected=False, exp_str='',
283                                   got_str='')
284         self.assertEquals(err.get(),
285                           ['  passes/image.html -> unexpected pass\n'])
286
287         printer, err, out = self.get_printer(['--print', 'nothing'])
288         printer.print_test_result(result, expected=False, exp_str='',
289                                   got_str='')
290         self.assertTrue(err.empty())
291
292         printer, err, out = self.get_printer(['--print',
293                                               'trace-unexpected'])
294         printer.print_test_result(result, expected=True, exp_str='',
295                                   got_str='')
296         self.assertTrue(err.empty())
297
298         printer, err, out = self.get_printer(['--print',
299                                               'trace-unexpected'])
300         printer.print_test_result(result, expected=False, exp_str='',
301                                   got_str='')
302         self.assertFalse(err.empty())
303
304         printer, err, out = self.get_printer(['--print',
305                                               'trace-unexpected'])
306         result = self.get_result("passes/text.html")
307         printer.print_test_result(result, expected=False, exp_str='',
308                                   got_str='')
309         self.assertFalse(err.empty())
310
311         err.reset()
312         printer.print_test_result(result, expected=False, exp_str='',
313                                   got_str='')
314         self.assertFalse(err.empty())
315
316         printer, err, out = self.get_printer(['--print', 'trace-everything'])
317         result = self.get_result('passes/image.html')
318         printer.print_test_result(result, expected=True, exp_str='',
319                                   got_str='')
320         result = self.get_result('failures/expected/missing_text.html')
321         printer.print_test_result(result, expected=True, exp_str='',
322                                   got_str='')
323         result = self.get_result('failures/expected/missing_check.html')
324         printer.print_test_result(result, expected=True, exp_str='',
325                                   got_str='')
326         result = self.get_result('failures/expected/missing_image.html')
327         printer.print_test_result(result, expected=True, exp_str='',
328                                   got_str='')
329         self.assertFalse(err.empty())
330
331         err.reset()
332         printer.print_test_result(result, expected=False, exp_str='',
333                                   got_str='')
334
335     def test_print_progress(self):
336         expectations = ''
337
338         # test that we print nothing
339         printer, err, out = self.get_printer(['--print', 'nothing'])
340         tests = ['passes/text.html', 'failures/expected/timeout.html',
341                  'failures/expected/crash.html']
342         paths, rs, exp = self.get_result_summary(tests, expectations)
343
344         printer.print_progress(rs, False, paths)
345         self.assertTrue(out.empty())
346         self.assertTrue(err.empty())
347
348         printer.print_progress(rs, True, paths)
349         self.assertTrue(out.empty())
350         self.assertTrue(err.empty())
351
352         # test regular functionality
353         printer, err, out = self.get_printer(['--print',
354                                               'one-line-progress'])
355         printer.print_progress(rs, False, paths)
356         self.assertTrue(out.empty())
357         self.assertFalse(err.empty())
358
359         err.reset()
360         out.reset()
361         printer.print_progress(rs, True, paths)
362         self.assertFalse(err.empty())
363         self.assertTrue(out.empty())
364
365     def test_print_progress__detailed(self):
366         tests = ['passes/text.html', 'failures/expected/timeout.html',
367                  'failures/expected/crash.html']
368         expectations = 'failures/expected/timeout.html = TIMEOUT'
369
370         # first, test that it is disabled properly
371         # should still print one-line-progress
372         printer, err, out = self.get_printer(
373             ['--print', 'detailed-progress'], single_threaded=False)
374         paths, rs, exp = self.get_result_summary(tests, expectations)
375         printer.print_progress(rs, False, paths)
376         self.assertFalse(err.empty())
377         self.assertTrue(out.empty())
378
379         # now test the enabled paths
380         printer, err, out = self.get_printer(
381             ['--print', 'detailed-progress'], single_threaded=True)
382         paths, rs, exp = self.get_result_summary(tests, expectations)
383         printer.print_progress(rs, False, paths)
384         self.assertFalse(err.empty())
385         self.assertTrue(out.empty())
386
387         err.reset()
388         out.reset()
389         printer.print_progress(rs, True, paths)
390         self.assertFalse(err.empty())
391         self.assertTrue(out.empty())
392
393         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
394         rs.add(self.get_result('failures/expected/timeout.html'), True)
395         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
396         err.reset()
397         out.reset()
398         printer.print_progress(rs, False, paths)
399         self.assertFalse(err.empty())
400         self.assertTrue(out.empty())
401
402         # We only clear the meter when retrying w/ detailed-progress.
403         err.reset()
404         out.reset()
405         printer.print_progress(rs, True, paths)
406         self.assertFalse(err.empty())
407         self.assertTrue(out.empty())
408
409         printer, err, out = self.get_printer(
410             ['--print', 'detailed-progress,unexpected'], single_threaded=True)
411         paths, rs, exp = self.get_result_summary(tests, expectations)
412         printer.print_progress(rs, False, paths)
413         self.assertFalse(err.empty())
414         self.assertTrue(out.empty())
415
416         err.reset()
417         out.reset()
418         printer.print_progress(rs, True, paths)
419         self.assertFalse(err.empty())
420         self.assertTrue(out.empty())
421
422         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
423         rs.add(self.get_result('failures/expected/timeout.html'), True)
424         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
425         err.reset()
426         out.reset()
427         printer.print_progress(rs, False, paths)
428         self.assertFalse(err.empty())
429         self.assertTrue(out.empty())
430
431         # We only clear the meter when retrying w/ detailed-progress.
432         err.reset()
433         out.reset()
434         printer.print_progress(rs, True, paths)
435         self.assertFalse(err.empty())
436         self.assertTrue(out.empty())
437
438     def test_write_nothing(self):
439         printer, err, out = self.get_printer(['--print', 'nothing'])
440         printer.write("foo")
441         self.assertTrue(err.empty())
442
443     def test_write_misc(self):
444         printer, err, out = self.get_printer(['--print', 'misc'])
445         printer.write("foo")
446         self.assertFalse(err.empty())
447         err.reset()
448         printer.write("foo", "config")
449         self.assertTrue(err.empty())
450
451     def test_write_everything(self):
452         printer, err, out = self.get_printer(['--print', 'everything'])
453         printer.write("foo")
454         self.assertFalse(err.empty())
455         err.reset()
456         printer.write("foo", "config")
457         self.assertFalse(err.empty())
458
459     def test_write_verbose(self):
460         printer, err, out = self.get_printer(['--verbose'])
461         printer.write("foo")
462         self.assertTrue(not err.empty() and "foo" in err.get()[0])
463         self.assertTrue(out.empty())
464
465     def test_print_unexpected_results(self):
466         # This routine is the only one that prints stuff that the bots
467         # care about.
468         #
469         # FIXME: there's some weird layering going on here. It seems
470         # like we shouldn't be both using an expectations string and
471         # having to specify whether or not the result was expected.
472         # This whole set of tests should probably be rewritten.
473         #
474         # FIXME: Plus, the fact that we're having to call into
475         # run_webkit_tests is clearly a layering inversion.
476         def get_unexpected_results(expected, passing, flaky):
477             """Return an unexpected results summary matching the input description.
478
479             There are a lot of different combinations of test results that
480             can be tested; this routine produces various combinations based
481             on the values of the input flags.
482
483             Args
484                 expected: whether the tests ran as expected
485                 passing: whether the tests should all pass
486                 flaky: whether the tests should be flaky (if False, they
487                     produce the same results on both runs; if True, they
488                     all pass on the second run).
489
490             """
491             paths, rs, exp = self.get_result_summary(tests, expectations)
492             if expected:
493                 rs.add(self.get_result('passes/text.html', test_expectations.PASS),
494                        expected)
495                 rs.add(self.get_result('failures/expected/timeout.html',
496                        test_expectations.TIMEOUT), expected)
497                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
498                    expected)
499             elif passing:
500                 rs.add(self.get_result('passes/text.html'), expected)
501                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
502                 rs.add(self.get_result('failures/expected/crash.html'), expected)
503             else:
504                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
505                        expected)
506                 rs.add(self.get_result('failures/expected/timeout.html',
507                        test_expectations.CRASH), expected)
508                 rs.add(self.get_result('failures/expected/crash.html',
509                                   test_expectations.TIMEOUT),
510                    expected)
511             retry = rs
512             if flaky:
513                 paths, retry, exp = self.get_result_summary(tests,
514                                                 expectations)
515                 retry.add(self.get_result('passes/text.html'), True)
516                 retry.add(self.get_result('failures/expected/timeout.html'), True)
517                 retry.add(self.get_result('failures/expected/crash.html'), True)
518             unexpected_results = run_webkit_tests.summarize_unexpected_results(
519                 self._port, exp, rs, retry)
520             return unexpected_results
521
522         tests = ['passes/text.html', 'failures/expected/timeout.html',
523                  'failures/expected/crash.html']
524         expectations = ''
525
526         printer, err, out = self.get_printer(['--print', 'nothing'])
527         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
528         printer.print_unexpected_results(ur)
529         self.assertTrue(err.empty())
530         self.assertTrue(out.empty())
531
532         printer, err, out = self.get_printer(['--print',
533                                               'unexpected-results'])
534
535         # test everything running as expected
536         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
537         printer.print_unexpected_results(ur)
538         self.assertTrue(err.empty())
539         self.assertTrue(out.empty())
540
541         # test failures
542         err.reset()
543         out.reset()
544         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
545         printer.print_unexpected_results(ur)
546         self.assertTrue(err.empty())
547         self.assertFalse(out.empty())
548
549         # test unexpected flaky results
550         err.reset()
551         out.reset()
552         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
553         printer.print_unexpected_results(ur)
554         self.assertTrue(err.empty())
555         self.assertFalse(out.empty())
556
557         # test unexpected passes
558         err.reset()
559         out.reset()
560         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
561         printer.print_unexpected_results(ur)
562         self.assertTrue(err.empty())
563         self.assertFalse(out.empty())
564
565         err.reset()
566         out.reset()
567         printer, err, out = self.get_printer(['--print', 'everything'])
568         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
569         printer.print_unexpected_results(ur)
570         self.assertTrue(err.empty())
571         self.assertFalse(out.empty())
572
573         expectations = """
574 failures/expected/crash.html = CRASH
575 failures/expected/timeout.html = TIMEOUT
576 """
577         err.reset()
578         out.reset()
579         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
580         printer.print_unexpected_results(ur)
581         self.assertTrue(err.empty())
582         self.assertFalse(out.empty())
583
584         err.reset()
585         out.reset()
586         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
587         printer.print_unexpected_results(ur)
588         self.assertTrue(err.empty())
589         self.assertFalse(out.empty())
590
591         # Test handling of --verbose as well.
592         err.reset()
593         out.reset()
594         printer, err, out = self.get_printer(['--verbose'])
595         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
596         printer.print_unexpected_results(ur)
597         self.assertTrue(err.empty())
598         self.assertFalse(out.empty())
599
600     def test_print_unexpected_results_buildbot(self):
601         # FIXME: Test that print_unexpected_results() produces the printer the
602         # buildbot is expecting.
603         pass
604
605 if __name__ == '__main__':
606     unittest.main()