Unreviewed, rolling out r112014.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / views / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import logging
33 import optparse
34 import StringIO
35 import time
36 import unittest
37
38 from webkitpy.common.host_mock import MockHost
39
40 from webkitpy.common.system import logtesting
41 from webkitpy.layout_tests import port
42 from webkitpy.layout_tests.controllers import manager
43 from webkitpy.layout_tests.models import result_summary
44 from webkitpy.layout_tests.models import test_expectations
45 from webkitpy.layout_tests.models import test_failures
46 from webkitpy.layout_tests.models import test_results
47 from webkitpy.layout_tests.views import printing
48
49
50 def get_options(args):
51     print_options = printing.print_options()
52     option_parser = optparse.OptionParser(option_list=print_options)
53     return option_parser.parse_args(args)
54
55
56 class TestUtilityFunctions(unittest.TestCase):
57     def assertEmpty(self, stream):
58         self.assertFalse(stream.getvalue())
59
60     def assertNotEmpty(self, stream):
61         self.assertTrue(stream.getvalue())
62
63     def assertWritten(self, stream, contents):
64         self.assertEquals(stream.buflist, contents)
65
66     def test_configure_logging(self):
67         options, args = get_options([])
68         stream = StringIO.StringIO()
69         handler = printing._configure_logging(stream, options.verbose)
70         logging.info("this should be logged")
71         self.assertNotEmpty(stream)
72
73         stream = StringIO.StringIO()
74         logging.debug("this should not be logged")
75         self.assertEmpty(stream)
76
77         printing._restore_logging(handler)
78
79         stream = StringIO.StringIO()
80         options, args = get_options(['--verbose'])
81         handler = printing._configure_logging(stream, options.verbose)
82         logging.debug("this should be logged")
83         self.assertNotEmpty(stream)
84         printing._restore_logging(handler)
85
86     def test_print_options(self):
87         options, args = get_options([])
88         self.assertTrue(options is not None)
89
90     def test_parse_print_options(self):
91         def test_switches(args, expected_switches_str, verbose=False):
92             options, args = get_options(args)
93             if expected_switches_str:
94                 expected_switches = set(expected_switches_str.split(','))
95             else:
96                 expected_switches = set()
97             switches = printing.parse_print_options(options.print_options,
98                                                     verbose)
99             self.assertEqual(expected_switches, switches)
100
101         # test that we default to the default set of switches
102         test_switches([], printing.PRINT_DEFAULT)
103
104         # test that verbose defaults to everything
105         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
106
107         # test that --print default does what it's supposed to
108         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
109
110         # test that --print nothing does what it's supposed to
111         test_switches(['--print', 'nothing'], None)
112
113         # test that --print everything does what it's supposed to
114         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
115
116         # this tests that '--print X' overrides '--verbose'
117         test_switches(['--print', 'actual'], 'actual', verbose=True)
118
119
120
121 class  Testprinter(unittest.TestCase):
122     def assertEmpty(self, stream):
123         self.assertFalse(stream.getvalue())
124
125     def assertNotEmpty(self, stream):
126         self.assertTrue(stream.getvalue())
127
128     def assertWritten(self, stream, contents):
129         self.assertEquals(stream.buflist, contents)
130
131     def reset(self, stream):
132         stream.buflist = []
133         stream.buf = ''
134
135     def get_printer(self, args=None, tty=False):
136         args = args or []
137         printing_options = printing.print_options()
138         option_parser = optparse.OptionParser(option_list=printing_options)
139         options, args = option_parser.parse_args(args)
140         host = MockHost()
141         self._port = host.port_factory.get('test', options)
142         nproc = 2
143
144         regular_output = StringIO.StringIO()
145         regular_output.isatty = lambda: tty
146         buildbot_output = StringIO.StringIO()
147         printer = printing.Printer(self._port, options, regular_output,
148                                    buildbot_output, configure_logging=True)
149         return printer, regular_output, buildbot_output
150
151     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
152         failures = []
153         if result_type == test_expectations.TIMEOUT:
154             failures = [test_failures.FailureTimeout()]
155         elif result_type == test_expectations.CRASH:
156             failures = [test_failures.FailureCrash()]
157         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
158
159     def get_result_summary(self, test_names, expectations_str):
160         expectations = test_expectations.TestExpectations(
161             self._port, test_names, expectations_str,
162             self._port.test_configuration(),
163             is_lint_mode=False)
164
165         rs = result_summary.ResultSummary(expectations, test_names)
166         return test_names, rs, expectations
167
168     def test_help_printer(self):
169         # Here and below we'll call the "regular" printer err and the
170         # buildbot printer out; this corresponds to how things run on the
171         # bots with stderr and stdout.
172         printer, err, out = self.get_printer()
173
174         # This routine should print something to stdout. testing what it is
175         # is kind of pointless.
176         printer.help_printing()
177         self.assertNotEmpty(err)
178         self.assertEmpty(out)
179
180     def do_switch_tests(self, method_name, switch, to_buildbot,
181                         message='hello', exp_err=None, exp_bot=None):
182         def do_helper(method_name, switch, message, exp_err, exp_bot):
183             printer, err, bot = self.get_printer(['--print', switch], tty=True)
184             getattr(printer, method_name)(message)
185             self.assertEqual(err.buflist, exp_err)
186             self.assertEqual(bot.buflist, exp_bot)
187
188         if to_buildbot:
189             if exp_err is None:
190                 exp_err = []
191             if exp_bot is None:
192                 exp_bot = [message + "\n"]
193         else:
194             if exp_err is None:
195                 exp_err = [message + "\n"]
196             if exp_bot is None:
197                 exp_bot = []
198         do_helper(method_name, 'nothing', 'hello', [], [])
199         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
200         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
201
202     def test_configure_and_cleanup(self):
203         # This test verifies that calling cleanup repeatedly and deleting
204         # the object is safe.
205         printer, err, out = self.get_printer(['--print', 'everything'])
206         printer.cleanup()
207         printer.cleanup()
208         printer = None
209
210     def test_print_actual(self):
211         # Actual results need to be logged to the buildbot's stream.
212         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
213
214     def test_print_actual_buildbot(self):
215         # FIXME: Test that the format of the actual results matches what the
216         # buildbot is expecting.
217         pass
218
219     def test_print_config(self):
220         self.do_switch_tests('print_config', 'config', to_buildbot=False)
221
222     def test_print_expected(self):
223         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
224
225     def test_print_timing(self):
226         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
227
228     def test_print_update(self):
229         # Note that there shouldn't be a carriage return here; updates()
230         # are meant to be overwritten.
231         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
232                              message='hello', exp_err=['hello'])
233
234     def test_print_one_line_summary(self):
235         printer, err, out = self.get_printer(['--print', 'nothing'])
236         printer.print_one_line_summary(1, 1, 0)
237         self.assertEmpty(err)
238
239         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
240         printer.print_one_line_summary(1, 1, 0)
241         self.assertWritten(err, ["All 1 tests ran as expected.\n", "\n"])
242
243         printer, err, out = self.get_printer(['--print', 'everything'])
244         printer.print_one_line_summary(1, 1, 0)
245         self.assertWritten(err, ["All 1 tests ran as expected.\n", "\n"])
246
247         printer, err, out = self.get_printer(['--print', 'everything'])
248         printer.print_one_line_summary(2, 1, 1)
249         self.assertWritten(err, ["1 test ran as expected, 1 didn't:\n", "\n"])
250
251         printer, err, out = self.get_printer(['--print', 'everything'])
252         printer.print_one_line_summary(3, 2, 1)
253         self.assertWritten(err, ["2 tests ran as expected, 1 didn't:\n", "\n"])
254
255         printer, err, out = self.get_printer(['--print', 'everything'])
256         printer.print_one_line_summary(3, 2, 0)
257         self.assertWritten(err, ['\n', "2 tests ran as expected (1 didn't run).\n", '\n'])
258
259
260     def test_print_test_result(self):
261         # Note here that we don't use meaningful exp_str and got_str values;
262         # the actual contents of the string are treated opaquely by
263         # print_test_result() when tracing, and usually we don't want
264         # to test what exactly is printed, just that something
265         # was printed (or that nothing was printed).
266         #
267         # FIXME: this is actually some goofy layering; it would be nice
268         # we could refactor it so that the args weren't redundant. Maybe
269         # the TestResult should contain what was expected, and the
270         # strings could be derived from the TestResult?
271         printer, err, out = self.get_printer(['--print', 'nothing'])
272         result = self.get_result('passes/image.html')
273         printer.print_test_result(result, expected=False, exp_str='',
274                                   got_str='')
275         self.assertEmpty(err)
276
277         printer, err, out = self.get_printer(['--print', 'unexpected'])
278         printer.print_test_result(result, expected=True, exp_str='',
279                                   got_str='')
280         self.assertEmpty(err)
281         printer.print_test_result(result, expected=False, exp_str='',
282                                   got_str='')
283         self.assertWritten(err, ['  passes/image.html -> unexpected pass\n'])
284
285         printer, err, out = self.get_printer(['--print', 'everything'])
286         printer.print_test_result(result, expected=True, exp_str='',
287                                   got_str='')
288         self.assertEmpty(err)
289
290         printer.print_test_result(result, expected=False, exp_str='',
291                                   got_str='')
292         self.assertWritten(err, ['  passes/image.html -> unexpected pass\n'])
293
294         printer, err, out = self.get_printer(['--print', 'nothing'])
295         printer.print_test_result(result, expected=False, exp_str='',
296                                   got_str='')
297         self.assertEmpty(err)
298
299         printer, err, out = self.get_printer(['--print',
300                                               'trace-unexpected'])
301         printer.print_test_result(result, expected=True, exp_str='',
302                                   got_str='')
303         self.assertEmpty(err)
304
305         printer, err, out = self.get_printer(['--print',
306                                               'trace-unexpected'])
307         printer.print_test_result(result, expected=False, exp_str='',
308                                   got_str='')
309         self.assertNotEmpty(err)
310
311         printer, err, out = self.get_printer(['--print',
312                                               'trace-unexpected'])
313         result = self.get_result("passes/text.html")
314         printer.print_test_result(result, expected=False, exp_str='',
315                                   got_str='')
316         self.assertNotEmpty(err)
317
318         printer, err, out = self.get_printer(['--print',
319                                               'trace-unexpected'])
320         result = self.get_result("passes/text.html")
321         printer.print_test_result(result, expected=False, exp_str='',
322                                   got_str='')
323         self.assertNotEmpty(err)
324
325         printer, err, out = self.get_printer(['--print', 'trace-everything'])
326         result = self.get_result('passes/image.html')
327         printer.print_test_result(result, expected=True, exp_str='',
328                                   got_str='')
329         result = self.get_result('failures/expected/missing_text.html')
330         printer.print_test_result(result, expected=True, exp_str='',
331                                   got_str='')
332         result = self.get_result('failures/expected/missing_check.html')
333         printer.print_test_result(result, expected=True, exp_str='',
334                                   got_str='')
335         result = self.get_result('failures/expected/missing_image.html')
336         printer.print_test_result(result, expected=True, exp_str='',
337                                   got_str='')
338         self.assertNotEmpty(err)
339
340         printer, err, out = self.get_printer(['--print', 'trace-everything'])
341         result = self.get_result('passes/image.html')
342         printer.print_test_result(result, expected=False, exp_str='',
343                                   got_str='')
344
345     def test_print_progress(self):
346         expectations = ''
347
348         printer, err, out = self.get_printer(['--print', 'nothing'])
349         tests = ['passes/text.html', 'failures/expected/timeout.html',
350                  'failures/expected/crash.html']
351         paths, rs, exp = self.get_result_summary(tests, expectations)
352
353         # First, test that we print nothing.
354         printer.print_progress(rs, False, paths)
355         self.assertEmpty(out)
356         self.assertEmpty(err)
357
358         printer.print_progress(rs, True, paths)
359         self.assertEmpty(out)
360         self.assertEmpty(err)
361
362         self.times = [1, 2, 12, 13, 14, 23, 33]
363
364         def mock_time():
365             return self.times.pop(0)
366
367         orig_time = time.time
368         try:
369             time.time = mock_time
370
371             # Test printing progress updates to a file.
372             printer, err, out = self.get_printer(['--print', 'one-line-progress'])
373             printer.print_progress(rs, False, paths)
374             printer.print_progress(rs, False, paths)
375             self.assertEmpty(out)
376             self.assertEmpty(err)
377
378             printer.print_progress(rs, False, paths)
379             self.assertEmpty(out)
380             self.assertNotEmpty(err)
381
382             self.reset(err)
383             self.reset(out)
384             printer.print_progress(rs, True, paths)
385             self.assertEmpty(out)
386             self.assertEmpty(err)
387
388             printer.print_progress(rs, True, paths)
389             self.assertEmpty(out)
390             self.assertNotEmpty(err)
391
392             # Now reconfigure the printer to test printing to a TTY instead of a file.
393             self.times = [1, 1.01, 2, 3]
394             printer, err, out = self.get_printer(['--print', 'one-line-progress'], tty=True)
395             printer.print_progress(rs, False, paths)
396             printer.print_progress(rs, False, paths)
397             self.assertEmpty(out)
398             self.assertEmpty(err)
399
400             printer.print_progress(rs, False, paths)
401             self.assertEmpty(out)
402             self.assertNotEmpty(err)
403
404             self.reset(err)
405             self.reset(out)
406             printer.print_progress(rs, True, paths)
407             self.assertEmpty(out)
408             self.assertNotEmpty(err)
409         finally:
410             time.time = orig_time
411
412     def test_write_nothing(self):
413         printer, err, out = self.get_printer(['--print', 'nothing'])
414         printer.write("foo")
415         self.assertEmpty(err)
416
417     def test_write_misc(self):
418         printer, err, out = self.get_printer(['--print', 'misc'])
419         printer.write("foo")
420         self.assertNotEmpty(err)
421
422         printer, err, out = self.get_printer(['--print', 'misc'])
423         printer.write("foo", "config")
424         self.assertEmpty(err)
425
426     def test_write_everything(self):
427         printer, err, out = self.get_printer(['--print', 'everything'])
428         printer.write("foo")
429         self.assertNotEmpty(err)
430
431         printer, err, out = self.get_printer(['--print', 'everything'])
432         printer.write("foo", "config")
433         self.assertNotEmpty(err)
434
435     def test_write_verbose(self):
436         printer, err, out = self.get_printer(['--verbose'])
437         printer.write("foo")
438         self.assertTrue("foo" in err.buflist[0])
439         self.assertEmpty(out)
440
441     def test_print_unexpected_results(self):
442         # This routine is the only one that prints stuff that the bots
443         # care about.
444         #
445         # FIXME: there's some weird layering going on here. It seems
446         # like we shouldn't be both using an expectations string and
447         # having to specify whether or not the result was expected.
448         # This whole set of tests should probably be rewritten.
449         #
450         # FIXME: Plus, the fact that we're having to call into
451         # run_webkit_tests is clearly a layering inversion.
452         def get_unexpected_results(expected, passing, flaky):
453             """Return an unexpected results summary matching the input description.
454
455             There are a lot of different combinations of test results that
456             can be tested; this routine produces various combinations based
457             on the values of the input flags.
458
459             Args
460                 expected: whether the tests ran as expected
461                 passing: whether the tests should all pass
462                 flaky: whether the tests should be flaky (if False, they
463                     produce the same results on both runs; if True, they
464                     all pass on the second run).
465
466             """
467             paths, rs, exp = self.get_result_summary(tests, expectations)
468             if expected:
469                 rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
470                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
471                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
472             elif passing:
473                 rs.add(self.get_result('passes/text.html'), expected)
474                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
475                 rs.add(self.get_result('failures/expected/crash.html'), expected)
476             else:
477                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
478                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
479                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
480             retry = rs
481             if flaky:
482                 paths, retry, exp = self.get_result_summary(tests, expectations)
483                 retry.add(self.get_result('passes/text.html'), True)
484                 retry.add(self.get_result('failures/expected/timeout.html'), True)
485                 retry.add(self.get_result('failures/expected/crash.html'), True)
486             unexpected_results = manager.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
487             return unexpected_results
488
489         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
490         expectations = ''
491
492         printer, err, out = self.get_printer(['--print', 'nothing'])
493         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
494         printer.print_unexpected_results(ur)
495         self.assertEmpty(err)
496         self.assertEmpty(out)
497
498         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
499
500         # test everything running as expected
501         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
502         printer.print_unexpected_results(ur)
503         self.assertEmpty(err)
504         self.assertEmpty(out)
505
506         # test failures
507         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
508         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
509         printer.print_unexpected_results(ur)
510         self.assertEmpty(err)
511         self.assertNotEmpty(out)
512
513         # test unexpected flaky
514         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
515         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
516         printer.print_unexpected_results(ur)
517         self.assertEmpty(err)
518         self.assertNotEmpty(out)
519
520         printer, err, out = self.get_printer(['--print', 'everything'])
521         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
522         printer.print_unexpected_results(ur)
523         self.assertEmpty(err)
524         self.assertNotEmpty(out)
525
526         expectations = """
527 BUGX : failures/expected/crash.html = CRASH
528 BUGX : failures/expected/timeout.html = TIMEOUT
529 """
530         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
531         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
532         printer.print_unexpected_results(ur)
533         self.assertEmpty(err)
534         self.assertNotEmpty(out)
535
536         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
537         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
538         printer.print_unexpected_results(ur)
539         self.assertEmpty(err)
540         self.assertNotEmpty(out)
541
542         # Test handling of --verbose as well.
543         printer, err, out = self.get_printer(['--verbose'])
544         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
545         printer.print_unexpected_results(ur)
546         # FIXME: debug output from the port and scm objects may or may not go
547         # to stderr, so there's no point in testing its contents here.
548         self.assertNotEmpty(out)
549
550     def test_print_unexpected_results_buildbot(self):
551         # FIXME: Test that print_unexpected_results() produces the printer the
552         # buildbot is expecting.
553         pass
554
555 if __name__ == '__main__':
556     unittest.main()