8777fd8d6e322ba6de2357c444c88ee6861dd33b
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / views / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010, 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import optparse
33 import StringIO
34 import time
35 import unittest
36
37 from webkitpy.common.host_mock import MockHost
38
39 from webkitpy.common.system import logtesting
40 from webkitpy.layout_tests import port
41 from webkitpy.layout_tests.controllers import manager
42 from webkitpy.layout_tests.models import result_summary
43 from webkitpy.layout_tests.models import test_expectations
44 from webkitpy.layout_tests.models import test_failures
45 from webkitpy.layout_tests.models import test_results
46 from webkitpy.layout_tests.views import printing
47
48
49 def get_options(args):
50     print_options = printing.print_options()
51     option_parser = optparse.OptionParser(option_list=print_options)
52     return option_parser.parse_args(args)
53
54
55 class TestUtilityFunctions(unittest.TestCase):
56     def test_print_options(self):
57         options, args = get_options([])
58         self.assertTrue(options is not None)
59
60     def test_parse_print_options(self):
61         def test_switches(args, expected_switches_str, verbose=False):
62             options, args = get_options(args)
63             if expected_switches_str:
64                 expected_switches = set(expected_switches_str.split(','))
65             else:
66                 expected_switches = set()
67             switches = printing.parse_print_options(options.print_options,
68                                                     verbose)
69             self.assertEqual(expected_switches, switches)
70
71         # test that we default to the default set of switches
72         test_switches([], printing.PRINT_DEFAULT)
73
74         # test that verbose defaults to everything
75         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
76
77         # test that --print default does what it's supposed to
78         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
79
80         # test that --print nothing does what it's supposed to
81         test_switches(['--print', 'nothing'], None)
82
83         # test that --print everything does what it's supposed to
84         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
85
86         # this tests that '--print X' overrides '--verbose'
87         test_switches(['--print', 'actual'], 'actual', verbose=True)
88
89
90
91 class  Testprinter(unittest.TestCase):
92     def assertEmpty(self, stream):
93         self.assertFalse(stream.getvalue())
94
95     def assertNotEmpty(self, stream):
96         self.assertTrue(stream.getvalue())
97
98     def assertWritten(self, stream, contents):
99         self.assertEquals(stream.buflist, contents)
100
101     def reset(self, stream):
102         stream.buflist = []
103         stream.buf = ''
104
105     def get_printer(self, args=None, tty=False):
106         args = args or []
107         printing_options = printing.print_options()
108         option_parser = optparse.OptionParser(option_list=printing_options)
109         options, args = option_parser.parse_args(args)
110         host = MockHost()
111         self._port = host.port_factory.get('test', options)
112         nproc = 2
113
114         regular_output = StringIO.StringIO()
115         regular_output.isatty = lambda: tty
116         buildbot_output = StringIO.StringIO()
117         printer = printing.Printer(self._port, options, regular_output, buildbot_output)
118         return printer, regular_output, buildbot_output
119
120     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
121         failures = []
122         if result_type == test_expectations.TIMEOUT:
123             failures = [test_failures.FailureTimeout()]
124         elif result_type == test_expectations.CRASH:
125             failures = [test_failures.FailureCrash()]
126         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
127
128     def get_result_summary(self, test_names, expectations_str):
129         expectations = test_expectations.TestExpectations(
130             self._port, test_names, expectations_str,
131             self._port.test_configuration(),
132             is_lint_mode=False)
133
134         rs = result_summary.ResultSummary(expectations, test_names)
135         return test_names, rs, expectations
136
137     def test_help_printer(self):
138         # Here and below we'll call the "regular" printer err and the
139         # buildbot printer out; this corresponds to how things run on the
140         # bots with stderr and stdout.
141         printer, err, out = self.get_printer()
142
143         # This routine should print something to stdout. testing what it is
144         # is kind of pointless.
145         printer.help_printing()
146         self.assertNotEmpty(err)
147         self.assertEmpty(out)
148
149     def do_switch_tests(self, method_name, switch, to_buildbot,
150                         message='hello', exp_err=None, exp_bot=None):
151         def do_helper(method_name, switch, message, exp_err, exp_bot):
152             printer, err, bot = self.get_printer(['--print', switch], tty=True)
153             getattr(printer, method_name)(message)
154             self.assertEqual(err.buflist, exp_err)
155             self.assertEqual(bot.buflist, exp_bot)
156
157         if to_buildbot:
158             if exp_err is None:
159                 exp_err = []
160             if exp_bot is None:
161                 exp_bot = [message + "\n"]
162         else:
163             if exp_err is None:
164                 exp_err = [message + "\n"]
165             if exp_bot is None:
166                 exp_bot = []
167         do_helper(method_name, 'nothing', 'hello', [], [])
168         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
169         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
170
171     def test_configure_and_cleanup(self):
172         # This test verifies that calling cleanup repeatedly and deleting
173         # the object is safe.
174         printer, err, out = self.get_printer(['--print', 'everything'])
175         printer.cleanup()
176         printer.cleanup()
177         printer = None
178
179     def test_print_actual(self):
180         # Actual results need to be logged to the buildbot's stream.
181         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
182
183     def test_print_actual_buildbot(self):
184         # FIXME: Test that the format of the actual results matches what the
185         # buildbot is expecting.
186         pass
187
188     def test_print_config(self):
189         self.do_switch_tests('print_config', 'config', to_buildbot=False)
190
191     def test_print_expected(self):
192         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
193
194     def test_print_timing(self):
195         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
196
197     def test_print_update(self):
198         # Note that there shouldn't be a carriage return here; updates()
199         # are meant to be overwritten.
200         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
201                              message='hello', exp_err=['hello'])
202
203     def test_print_one_line_summary(self):
204         printer, err, out = self.get_printer(['--print', 'nothing'])
205         printer.print_one_line_summary(1, 1, 0)
206         self.assertEmpty(err)
207
208         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
209         printer.print_one_line_summary(1, 1, 0)
210         self.assertWritten(err, ["All 1 tests ran as expected.\n", "\n"])
211
212         printer, err, out = self.get_printer(['--print', 'everything'])
213         printer.print_one_line_summary(1, 1, 0)
214         self.assertWritten(err, ["All 1 tests ran as expected.\n", "\n"])
215
216         printer, err, out = self.get_printer(['--print', 'everything'])
217         printer.print_one_line_summary(2, 1, 1)
218         self.assertWritten(err, ["1 test ran as expected, 1 didn't:\n", "\n"])
219
220         printer, err, out = self.get_printer(['--print', 'everything'])
221         printer.print_one_line_summary(3, 2, 1)
222         self.assertWritten(err, ["2 tests ran as expected, 1 didn't:\n", "\n"])
223
224         printer, err, out = self.get_printer(['--print', 'everything'])
225         printer.print_one_line_summary(3, 2, 0)
226         self.assertWritten(err, ['\n', "2 tests ran as expected (1 didn't run).\n", '\n'])
227
228
229     def test_print_test_result(self):
230         # Note here that we don't use meaningful exp_str and got_str values;
231         # the actual contents of the string are treated opaquely by
232         # print_test_result() when tracing, and usually we don't want
233         # to test what exactly is printed, just that something
234         # was printed (or that nothing was printed).
235         #
236         # FIXME: this is actually some goofy layering; it would be nice
237         # we could refactor it so that the args weren't redundant. Maybe
238         # the TestResult should contain what was expected, and the
239         # strings could be derived from the TestResult?
240         printer, err, out = self.get_printer(['--print', 'nothing'])
241         result = self.get_result('passes/image.html')
242         printer.print_test_result(result, expected=False, exp_str='',
243                                   got_str='')
244         self.assertEmpty(err)
245
246         printer, err, out = self.get_printer(['--print', 'unexpected'])
247         printer.print_test_result(result, expected=True, exp_str='',
248                                   got_str='')
249         self.assertEmpty(err)
250         printer.print_test_result(result, expected=False, exp_str='',
251                                   got_str='')
252         self.assertWritten(err, ['  passes/image.html -> unexpected pass\n'])
253
254         printer, err, out = self.get_printer(['--print', 'everything'])
255         printer.print_test_result(result, expected=True, exp_str='',
256                                   got_str='')
257         self.assertEmpty(err)
258
259         printer.print_test_result(result, expected=False, exp_str='',
260                                   got_str='')
261         self.assertWritten(err, ['  passes/image.html -> unexpected pass\n'])
262
263         printer, err, out = self.get_printer(['--print', 'nothing'])
264         printer.print_test_result(result, expected=False, exp_str='',
265                                   got_str='')
266         self.assertEmpty(err)
267
268         printer, err, out = self.get_printer(['--print',
269                                               'trace-unexpected'])
270         printer.print_test_result(result, expected=True, exp_str='',
271                                   got_str='')
272         self.assertEmpty(err)
273
274         printer, err, out = self.get_printer(['--print',
275                                               'trace-unexpected'])
276         printer.print_test_result(result, expected=False, exp_str='',
277                                   got_str='')
278         self.assertNotEmpty(err)
279
280         printer, err, out = self.get_printer(['--print',
281                                               'trace-unexpected'])
282         result = self.get_result("passes/text.html")
283         printer.print_test_result(result, expected=False, exp_str='',
284                                   got_str='')
285         self.assertNotEmpty(err)
286
287         printer, err, out = self.get_printer(['--print',
288                                               'trace-unexpected'])
289         result = self.get_result("passes/text.html")
290         printer.print_test_result(result, expected=False, exp_str='',
291                                   got_str='')
292         self.assertNotEmpty(err)
293
294         printer, err, out = self.get_printer(['--print', 'trace-everything'])
295         result = self.get_result('passes/image.html')
296         printer.print_test_result(result, expected=True, exp_str='',
297                                   got_str='')
298         result = self.get_result('failures/expected/missing_text.html')
299         printer.print_test_result(result, expected=True, exp_str='',
300                                   got_str='')
301         result = self.get_result('failures/expected/missing_check.html')
302         printer.print_test_result(result, expected=True, exp_str='',
303                                   got_str='')
304         result = self.get_result('failures/expected/missing_image.html')
305         printer.print_test_result(result, expected=True, exp_str='',
306                                   got_str='')
307         self.assertNotEmpty(err)
308
309         printer, err, out = self.get_printer(['--print', 'trace-everything'])
310         result = self.get_result('passes/image.html')
311         printer.print_test_result(result, expected=False, exp_str='',
312                                   got_str='')
313
314     def test_print_progress(self):
315         expectations = ''
316
317         printer, err, out = self.get_printer(['--print', 'nothing'])
318         tests = ['passes/text.html', 'failures/expected/timeout.html',
319                  'failures/expected/crash.html']
320         paths, rs, exp = self.get_result_summary(tests, expectations)
321
322         # First, test that we print nothing when we shouldn't print anything.
323         printer.print_progress(rs, False, paths)
324         self.assertEmpty(out)
325         self.assertEmpty(err)
326
327         printer.print_progress(rs, True, paths)
328         self.assertEmpty(out)
329         self.assertEmpty(err)
330
331         # Now test that we do print things.
332         printer, err, out = self.get_printer(['--print', 'one-line-progress'])
333         printer.print_progress(rs, False, paths)
334         self.assertEmpty(out)
335         self.assertNotEmpty(err)
336
337         printer, err, out = self.get_printer(['--print', 'one-line-progress'])
338         printer.print_progress(rs, True, paths)
339         self.assertEmpty(out)
340         self.assertNotEmpty(err)
341
342         printer, err, out = self.get_printer(['--print', 'one-line-progress'])
343         rs.remaining = 0
344         printer.print_progress(rs, False, paths)
345         self.assertEmpty(out)
346         self.assertNotEmpty(err)
347
348         printer.print_progress(rs, True, paths)
349         self.assertEmpty(out)
350         self.assertNotEmpty(err)
351
352
353
354     def test_write_nothing(self):
355         printer, err, out = self.get_printer(['--print', 'nothing'])
356         printer.write("foo")
357         self.assertEmpty(err)
358
359     def test_write_misc(self):
360         printer, err, out = self.get_printer(['--print', 'misc'])
361         printer.write("foo")
362         self.assertNotEmpty(err)
363
364         printer, err, out = self.get_printer(['--print', 'misc'])
365         printer.write("foo", "config")
366         self.assertEmpty(err)
367
368     def test_write_everything(self):
369         printer, err, out = self.get_printer(['--print', 'everything'])
370         printer.write("foo")
371         self.assertNotEmpty(err)
372
373         printer, err, out = self.get_printer(['--print', 'everything'])
374         printer.write("foo", "config")
375         self.assertNotEmpty(err)
376
377     def test_write_verbose(self):
378         printer, err, out = self.get_printer(['--verbose'])
379         printer.write("foo")
380         self.assertTrue("foo" in err.buflist[0])
381         self.assertEmpty(out)
382
383     def test_print_unexpected_results(self):
384         # This routine is the only one that prints stuff that the bots
385         # care about.
386         #
387         # FIXME: there's some weird layering going on here. It seems
388         # like we shouldn't be both using an expectations string and
389         # having to specify whether or not the result was expected.
390         # This whole set of tests should probably be rewritten.
391         #
392         # FIXME: Plus, the fact that we're having to call into
393         # run_webkit_tests is clearly a layering inversion.
394         def get_unexpected_results(expected, passing, flaky):
395             """Return an unexpected results summary matching the input description.
396
397             There are a lot of different combinations of test results that
398             can be tested; this routine produces various combinations based
399             on the values of the input flags.
400
401             Args
402                 expected: whether the tests ran as expected
403                 passing: whether the tests should all pass
404                 flaky: whether the tests should be flaky (if False, they
405                     produce the same results on both runs; if True, they
406                     all pass on the second run).
407
408             """
409             paths, rs, exp = self.get_result_summary(tests, expectations)
410             if expected:
411                 rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
412                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
413                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
414             elif passing:
415                 rs.add(self.get_result('passes/text.html'), expected)
416                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
417                 rs.add(self.get_result('failures/expected/crash.html'), expected)
418             else:
419                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
420                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
421                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
422             retry = rs
423             if flaky:
424                 paths, retry, exp = self.get_result_summary(tests, expectations)
425                 retry.add(self.get_result('passes/text.html'), True)
426                 retry.add(self.get_result('failures/expected/timeout.html'), True)
427                 retry.add(self.get_result('failures/expected/crash.html'), True)
428             unexpected_results = manager.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
429             return unexpected_results
430
431         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
432         expectations = ''
433
434         printer, err, out = self.get_printer(['--print', 'nothing'])
435         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
436         printer.print_unexpected_results(ur)
437         self.assertEmpty(err)
438         self.assertEmpty(out)
439
440         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
441
442         # test everything running as expected
443         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
444         printer.print_unexpected_results(ur)
445         self.assertEmpty(err)
446         self.assertEmpty(out)
447
448         # test failures
449         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
450         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
451         printer.print_unexpected_results(ur)
452         self.assertEmpty(err)
453         self.assertNotEmpty(out)
454
455         # test unexpected flaky
456         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
457         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
458         printer.print_unexpected_results(ur)
459         self.assertEmpty(err)
460         self.assertNotEmpty(out)
461
462         printer, err, out = self.get_printer(['--print', 'everything'])
463         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
464         printer.print_unexpected_results(ur)
465         self.assertEmpty(err)
466         self.assertNotEmpty(out)
467
468         expectations = """
469 BUGX : failures/expected/crash.html = CRASH
470 BUGX : failures/expected/timeout.html = TIMEOUT
471 """
472         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
473         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
474         printer.print_unexpected_results(ur)
475         self.assertEmpty(err)
476         self.assertNotEmpty(out)
477
478         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
479         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
480         printer.print_unexpected_results(ur)
481         self.assertEmpty(err)
482         self.assertNotEmpty(out)
483
484         # Test handling of --verbose as well.
485         printer, err, out = self.get_printer(['--verbose'])
486         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
487         printer.print_unexpected_results(ur)
488         # FIXME: debug output from the port and scm objects may or may not go
489         # to stderr, so there's no point in testing its contents here.
490         self.assertNotEmpty(out)
491
492     def test_print_unexpected_results_buildbot(self):
493         # FIXME: Test that print_unexpected_results() produces the printer the
494         # buildbot is expecting.
495         pass
496
497 if __name__ == '__main__':
498     unittest.main()