c8648bcfc86f008e3494e9048411c8f1ba2090c6
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / printing_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for printing.py."""
31
32 import os
33 import optparse
34 import pdb
35 import sys
36 import unittest
37 import logging
38
39 from webkitpy.common import array_stream
40 from webkitpy.layout_tests import port
41 from webkitpy.layout_tests.layout_package import printing
42 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
43 from webkitpy.layout_tests.layout_package import test_expectations
44 from webkitpy.layout_tests.layout_package import test_failures
45 from webkitpy.layout_tests import run_webkit_tests
46
47
48 def get_options(args):
49     print_options = printing.print_options()
50     option_parser = optparse.OptionParser(option_list=print_options)
51     return option_parser.parse_args(args)
52
53
54 def get_result(filename, result_type=test_expectations.PASS, run_time=0):
55     failures = []
56     if result_type == test_expectations.TIMEOUT:
57         failures = [test_failures.FailureTimeout()]
58     elif result_type == test_expectations.CRASH:
59         failures = [test_failures.FailureCrash()]
60     return dump_render_tree_thread.TestResult(filename, failures, run_time,
61                                               total_time_for_all_diffs=0,
62                                               time_for_diffs=0)
63
64
65 def get_result_summary(port_obj, test_files, expectations_str):
66     expectations = test_expectations.TestExpectations(
67         port_obj, test_files, expectations_str,
68         port_obj.test_platform_name(), is_debug_mode=False,
69         is_lint_mode=False, tests_are_present=False)
70
71     rs = run_webkit_tests.ResultSummary(expectations, test_files)
72     return rs, expectations
73
74
75 class TestUtilityFunctions(unittest.TestCase):
76     def test_configure_logging(self):
77         # FIXME: We need to figure out how to reset the basic logger.
78         # FIXME: If other testing classes call logging.basicConfig() then
79         # FIXME: these calls become no-ops and we can't control the
80         # FIXME: configuration to test things properly.
81         options, args = get_options([])
82         stream = array_stream.ArrayStream()
83         printing.configure_logging(options, stream)
84         logging.info("this should be logged")
85         # self.assertFalse(stream.empty())
86
87         stream.reset()
88         logging.debug("this should not be logged")
89         # self.assertTrue(stream.empty())
90
91         stream.reset()
92         options, args = get_options(['--verbose'])
93         printing.configure_logging(options, stream)
94         logging.debug("this should be logged")
95         # self.assertFalse(stream.empty())
96
97     def test_print_options(self):
98         options, args = get_options([])
99         self.assertTrue(options is not None)
100
101     def test_parse_print_options(self):
102         def test_switches(args, verbose, child_processes, is_fully_parallel,
103                           expected_switches_str):
104             options, args = get_options(args)
105             if expected_switches_str:
106                 expected_switches = set(expected_switches_str.split(','))
107             else:
108                 expected_switches = set()
109             switches = printing.parse_print_options(options.print_options,
110                                                     verbose,
111                                                     child_processes,
112                                                     is_fully_parallel)
113             self.assertEqual(expected_switches, switches)
114
115         # test that we default to the default set of switches
116         test_switches([], False, 1, False,
117                       printing.PRINT_DEFAULT)
118
119         # test that verbose defaults to everything
120         test_switches([], True, 1, False,
121                       printing.PRINT_EVERYTHING)
122
123         # test that --print default does what it's supposed to
124         test_switches(['--print', 'default'], False, 1, False,
125                       printing.PRINT_DEFAULT)
126
127         # test that --print nothing does what it's supposed to
128         test_switches(['--print', 'nothing'], False, 1, False,
129                       None)
130
131         # test that --print everything does what it's supposed to
132         test_switches(['--print', 'everything'], False, 1, False,
133                       printing.PRINT_EVERYTHING)
134
135         # this tests that '--print X' overrides '--verbose'
136         test_switches(['--print', 'actual'], True, 1, False,
137                       'actual')
138
139
140 class  Testprinter(unittest.TestCase):
141     def get_printer(self, args=None, single_threaded=False,
142                    is_fully_parallel=False):
143         printing_options = printing.print_options()
144         option_parser = optparse.OptionParser(option_list=printing_options)
145         options, args = option_parser.parse_args(args)
146         self._port = port.get('test', options)
147         nproc = 2
148         if single_threaded:
149             nproc = 1
150
151         regular_output = array_stream.ArrayStream()
152         buildbot_output = array_stream.ArrayStream()
153         printer = printing.Printer(self._port, options, regular_output,
154                                    buildbot_output, single_threaded,
155                                    is_fully_parallel)
156         return printer, regular_output, buildbot_output
157
158     def test_help_printer(self):
159         # Here and below we'll call the "regular" printer err and the
160         # buildbot printer out; this corresponds to how things run on the
161         # bots with stderr and stdout.
162         printer, err, out = self.get_printer()
163
164         # This routine should print something to stdout. testing what it is
165         # is kind of pointless.
166         printer.help_printing()
167         self.assertFalse(err.empty())
168         self.assertTrue(out.empty())
169
170     def do_switch_tests(self, method_name, switch, to_buildbot,
171                         message='hello', exp_err=None, exp_bot=None):
172         def do_helper(method_name, switch, message, exp_err, exp_bot):
173             printer, err, bot = self.get_printer(['--print', switch])
174             getattr(printer, method_name)(message)
175             self.assertEqual(err.get(), exp_err)
176             self.assertEqual(bot.get(), exp_bot)
177
178         if to_buildbot:
179             if exp_err is None:
180                 exp_err = []
181             if exp_bot is None:
182                 exp_bot = [message + "\n"]
183         else:
184             if exp_err is None:
185                 exp_err = [message + "\n"]
186             if exp_bot is None:
187                 exp_bot = []
188         do_helper(method_name, 'nothing', 'hello', [], [])
189         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
190         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
191
192     def test_print_actual(self):
193         # Actual results need to be logged to the buildbot's stream.
194         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
195
196     def test_print_actual_buildbot(self):
197         # FIXME: Test that the format of the actual results matches what the
198         # buildbot is expecting.
199         pass
200
201     def test_print_config(self):
202         self.do_switch_tests('print_config', 'config', to_buildbot=False)
203
204     def test_print_expected(self):
205         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
206
207     def test_print_timing(self):
208         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
209
210     def test_print_update(self):
211         # Note that there shouldn't be a carriage return here; updates()
212         # are meant to be overwritten.
213         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
214                              message='hello', exp_err=['hello'])
215
216     def test_print_one_line_summary(self):
217         printer, err, out = self.get_printer(['--print', 'nothing'])
218         printer.print_one_line_summary(1, 1, 0)
219         self.assertTrue(err.empty())
220
221         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
222         printer.print_one_line_summary(1, 1, 0)
223         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
224
225         printer, err, out = self.get_printer(['--print', 'everything'])
226         printer.print_one_line_summary(1, 1, 0)
227         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
228
229         err.reset()
230         printer.print_one_line_summary(2, 1, 1)
231         self.assertEquals(err.get(),
232                           ["1 test ran as expected, 1 didn't:\n", "\n"])
233
234         err.reset()
235         printer.print_one_line_summary(3, 2, 1)
236         self.assertEquals(err.get(),
237                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
238
239         err.reset()
240         printer.print_one_line_summary(3, 2, 0)
241         self.assertEquals(err.get(),
242                           ['\n', "2 tests ran as expected (1 didn't run).\n",
243                            '\n'])
244
245
246     def test_print_test_result(self):
247         result = get_result('foo.html')
248         printer, err, out = self.get_printer(['--print', 'nothing'])
249         result = get_result(os.path.join(self._port.layout_tests_dir(),
250                                          'foo.html'))
251         printer.print_test_result(result, expected=False, exp_str='',
252                                   got_str='')
253         self.assertTrue(err.empty())
254
255         printer, err, out = self.get_printer(['--print', 'unexpected'])
256         printer.print_test_result(result, expected=True, exp_str='',
257                                   got_str='')
258         self.assertTrue(err.empty())
259         printer.print_test_result(result, expected=False, exp_str='',
260                                   got_str='')
261         self.assertEquals(err.get(),
262                           ['  foo.html -> unexpected pass\n'])
263
264         printer, err, out = self.get_printer(['--print', 'everything'])
265         printer.print_test_result(result, expected=True, exp_str='',
266                                   got_str='')
267         self.assertTrue(err.empty())
268
269         printer.print_test_result(result, expected=False, exp_str='',
270                                   got_str='')
271         self.assertEquals(err.get(),
272                           ['  foo.html -> unexpected pass\n'])
273
274         printer, err, out = self.get_printer(['--print', 'nothing'])
275         printer.print_test_result(result, expected=False, exp_str='',
276                                   got_str='')
277         self.assertTrue(err.empty())
278
279         printer, err, out = self.get_printer(['--print',
280                                                  'trace-unexpected'])
281         printer.print_test_result(result, expected=True, exp_str='',
282                                   got_str='')
283         self.assertTrue(err.empty())
284
285         err.reset()
286         printer.print_test_result(result, expected=False, exp_str='',
287                                   got_str='')
288         self.assertFalse(err.empty())
289
290         printer, err, out = self.get_printer(['--print', 'trace-everything'])
291         printer.print_test_result(result, expected=True, exp_str='',
292                                   got_str='')
293         self.assertFalse(err.empty())
294
295         err.reset()
296         printer.print_test_result(result, expected=False, exp_str='',
297                                   got_str='')
298
299     def test_print_progress(self):
300         test_files = ['foo.html', 'bar.html']
301         expectations = ''
302
303         # test that we print nothing
304         printer, err, out = self.get_printer(['--print', 'nothing'])
305         rs, exp = get_result_summary(self._port, test_files, expectations)
306
307         printer.print_progress(rs, False, test_files)
308         self.assertTrue(out.empty())
309         self.assertTrue(err.empty())
310
311         printer.print_progress(rs, True, test_files)
312         self.assertTrue(out.empty())
313         self.assertTrue(err.empty())
314
315         # test regular functionality
316         printer, err, out = self.get_printer(['--print',
317                                               'one-line-progress'])
318         printer.print_progress(rs, False, test_files)
319         self.assertTrue(out.empty())
320         self.assertFalse(err.empty())
321
322         err.reset()
323         out.reset()
324         printer.print_progress(rs, True, test_files)
325         self.assertFalse(err.empty())
326         self.assertTrue(out.empty())
327
328     def test_print_progress__detailed(self):
329         test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html']
330         expectations = 'pass/timeout.html = TIMEOUT'
331
332         # first, test that it is disabled properly
333         # should still print one-line-progress
334         printer, err, out = self.get_printer(
335             ['--print', 'detailed-progress'], single_threaded=False)
336         rs, exp = get_result_summary(self._port, test_files, expectations)
337         printer.print_progress(rs, False, test_files)
338         self.assertFalse(err.empty())
339         self.assertTrue(out.empty())
340
341         # now test the enabled paths
342         printer, err, out = self.get_printer(
343             ['--print', 'detailed-progress'], single_threaded=True)
344         rs, exp = get_result_summary(self._port, test_files, expectations)
345         printer.print_progress(rs, False, test_files)
346         self.assertFalse(err.empty())
347         self.assertTrue(out.empty())
348
349         err.reset()
350         out.reset()
351         printer.print_progress(rs, True, test_files)
352         self.assertFalse(err.empty())
353         self.assertTrue(out.empty())
354
355         rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False)
356         rs.add(get_result('pass/timeout.html'), True)
357         rs.add(get_result('fail/crash.html', test_expectations.CRASH), True)
358         err.reset()
359         out.reset()
360         printer.print_progress(rs, False, test_files)
361         self.assertFalse(err.empty())
362         self.assertTrue(out.empty())
363
364         # We only clear the meter when retrying w/ detailed-progress.
365         err.reset()
366         out.reset()
367         printer.print_progress(rs, True, test_files)
368         self.assertEqual(err.get(), [''])
369         self.assertTrue(out.empty())
370
371         printer, err, out = self.get_printer(
372             ['--print', 'detailed-progress,unexpected'], single_threaded=True)
373         rs, exp = get_result_summary(self._port, test_files, expectations)
374         printer.print_progress(rs, False, test_files)
375         self.assertFalse(err.empty())
376         self.assertTrue(out.empty())
377
378         err.reset()
379         out.reset()
380         printer.print_progress(rs, True, test_files)
381         self.assertFalse(err.empty())
382         self.assertTrue(out.empty())
383
384         rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False)
385         rs.add(get_result('pass/timeout.html'), True)
386         rs.add(get_result('fail/crash.html', test_expectations.CRASH), True)
387         err.reset()
388         out.reset()
389         printer.print_progress(rs, False, test_files)
390         self.assertFalse(err.empty())
391         self.assertTrue(out.empty())
392
393         # We only clear the meter when retrying w/ detailed-progress.
394         err.reset()
395         out.reset()
396         printer.print_progress(rs, True, test_files)
397         self.assertEqual(err.get(), [''])
398         self.assertTrue(out.empty())
399
400     def test_write(self):
401         printer, err, out = self.get_printer(['--print', 'nothing'])
402         printer.write("foo")
403         self.assertTrue(err.empty())
404
405         printer, err, out = self.get_printer(['--print', 'misc'])
406         printer.write("foo")
407         self.assertFalse(err.empty())
408         err.reset()
409         printer.write("foo", "config")
410         self.assertTrue(err.empty())
411
412         printer, err, out = self.get_printer(['--print', 'everything'])
413         printer.write("foo")
414         self.assertFalse(err.empty())
415         err.reset()
416         printer.write("foo", "config")
417         self.assertFalse(err.empty())
418
419     def test_print_unexpected_results(self):
420         # This routine is the only one that prints stuff that the bots
421         # care about.
422         def get_unexpected_results(expected, passing, flaky):
423             rs, exp = get_result_summary(self._port, test_files, expectations)
424             if expected:
425                 rs.add(get_result('pass/pass.html', test_expectations.PASS),
426                        expected)
427                 rs.add(get_result('pass/timeout.html',
428                        test_expectations.TIMEOUT), expected)
429                 rs.add(get_result('fail/crash.html', test_expectations.CRASH),
430                    expected)
431             elif passing:
432                 rs.add(get_result('pass/pass.html'), expected)
433                 rs.add(get_result('pass/timeout.html'), expected)
434                 rs.add(get_result('fail/crash.html'), expected)
435             else:
436                 rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT),
437                        expected)
438                 rs.add(get_result('pass/timeout.html',
439                        test_expectations.CRASH), expected)
440                 rs.add(get_result('fail/crash.html',
441                                   test_expectations.TIMEOUT),
442                    expected)
443             retry = rs
444             if flaky:
445                 retry, exp = get_result_summary(self._port, test_files,
446                                                 expectations)
447                 retry.add(get_result('pass/pass.html'), True)
448                 retry.add(get_result('pass/timeout.html'), True)
449                 retry.add(get_result('fail/crash.html'), True)
450             unexpected_results = run_webkit_tests.summarize_unexpected_results(
451                 self._port, exp, rs, retry)
452             return unexpected_results
453
454         test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html']
455         expectations = ''
456
457         printer, err, out = self.get_printer(['--print', 'nothing'])
458         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
459         printer.print_unexpected_results(ur)
460         self.assertTrue(err.empty())
461         self.assertTrue(out.empty())
462
463         printer, err, out = self.get_printer(['--print',
464                                               'unexpected-results'])
465
466         # test everything running as expected
467         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
468         printer.print_unexpected_results(ur)
469         self.assertTrue(err.empty())
470         self.assertTrue(out.empty())
471
472         # test failures
473         err.reset()
474         out.reset()
475         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
476         printer.print_unexpected_results(ur)
477         self.assertTrue(err.empty())
478         self.assertFalse(out.empty())
479
480         # test unexpected flaky results
481         err.reset()
482         out.reset()
483         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
484         printer.print_unexpected_results(ur)
485         self.assertTrue(err.empty())
486         self.assertFalse(out.empty())
487
488         # test unexpected passes
489         err.reset()
490         out.reset()
491         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
492         printer.print_unexpected_results(ur)
493         self.assertTrue(err.empty())
494         self.assertFalse(out.empty())
495
496         err.reset()
497         out.reset()
498         printer, err, out = self.get_printer(['--print', 'everything'])
499         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
500         printer.print_unexpected_results(ur)
501         self.assertTrue(err.empty())
502         self.assertFalse(out.empty())
503
504     def test_print_unexpected_results_buildbot(self):
505         # FIXME: Test that print_unexpected_results() produces the printer the
506         # buildbot is expecting.
507         pass
508
509 if __name__ == '__main__':
510     unittest.main()