Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Unit tests for printing.py."""
  31
  32 import os
  33 import optparse
  34 import pdb
  35 import sys
  36 import unittest
  37 import logging
  38
  39 from webkitpy.common import array_stream
  40 from webkitpy.common.system import logtesting
  41 from webkitpy.layout_tests import port
  42 from webkitpy.layout_tests.layout_package import printing
  43 from webkitpy.layout_tests.layout_package import test_results
  44 from webkitpy.layout_tests.layout_package import test_expectations
  45 from webkitpy.layout_tests.layout_package import test_failures
  46 from webkitpy.layout_tests import run_webkit_tests
  47
  48
  49 def get_options(args):
  50     print_options = printing.print_options()
  51     option_parser = optparse.OptionParser(option_list=print_options)
  52     return option_parser.parse_args(args)
  53
  54
  55 class TestUtilityFunctions(unittest.TestCase):
  56     def test_configure_logging(self):
  57         options, args = get_options([])
  58         stream = array_stream.ArrayStream()
  59         handler = printing._configure_logging(stream, options.verbose)
  60         logging.info("this should be logged")
  61         self.assertFalse(stream.empty())
  62
  63         stream.reset()
  64         logging.debug("this should not be logged")
  65         self.assertTrue(stream.empty())
  66
  67         printing._restore_logging(handler)
  68
  69         stream.reset()
  70         options, args = get_options(['--verbose'])
  71         handler = printing._configure_logging(stream, options.verbose)
  72         logging.debug("this should be logged")
  73         self.assertFalse(stream.empty())
  74         printing._restore_logging(handler)
  75
  76     def test_print_options(self):
  77         options, args = get_options([])
  78         self.assertTrue(options is not None)
  79
  80     def test_parse_print_options(self):
  81         def test_switches(args, expected_switches_str,
  82                           verbose=False, child_processes=1,
  83                           is_fully_parallel=False):
  84             options, args = get_options(args)
  85             if expected_switches_str:
  86                 expected_switches = set(expected_switches_str.split(','))
  87             else:
  88                 expected_switches = set()
  89             switches = printing.parse_print_options(options.print_options,
  90                                                     verbose,
  91                                                     child_processes,
  92                                                     is_fully_parallel)
  93             self.assertEqual(expected_switches, switches)
  94
  95         # test that we default to the default set of switches
  96         test_switches([], printing.PRINT_DEFAULT)
  97
  98         # test that verbose defaults to everything
  99         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
 100
 101         # test that --print default does what it's supposed to
 102         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
 103
 104         # test that --print nothing does what it's supposed to
 105         test_switches(['--print', 'nothing'], None)
 106
 107         # test that --print everything does what it's supposed to
 108         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
 109
 110         # this tests that '--print X' overrides '--verbose'
 111         test_switches(['--print', 'actual'], 'actual', verbose=True)
 112
 113
 114
 115 class  Testprinter(unittest.TestCase):
 116     def get_printer(self, args=None, single_threaded=False,
 117                    is_fully_parallel=False):
 118         printing_options = printing.print_options()
 119         option_parser = optparse.OptionParser(option_list=printing_options)
 120         options, args = option_parser.parse_args(args)
 121         self._port = port.get('test', options)
 122         nproc = 2
 123         if single_threaded:
 124             nproc = 1
 125
 126         regular_output = array_stream.ArrayStream()
 127         buildbot_output = array_stream.ArrayStream()
 128         printer = printing.Printer(self._port, options, regular_output,
 129                                    buildbot_output, single_threaded,
 130                                    is_fully_parallel)
 131         return printer, regular_output, buildbot_output
 132
 133     def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
 134         failures = []
 135         if result_type == test_expectations.TIMEOUT:
 136             failures = [test_failures.FailureTimeout()]
 137         elif result_type == test_expectations.CRASH:
 138             failures = [test_failures.FailureCrash()]
 139         path = os.path.join(self._port.layout_tests_dir(), test)
 140         return test_results.TestResult(path, failures, run_time,
 141                                        total_time_for_all_diffs=0,
 142                                        time_for_diffs=0)
 143
 144     def get_result_summary(self, tests, expectations_str):
 145         test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
 146                       test in tests]
 147         expectations = test_expectations.TestExpectations(
 148             self._port, test_paths, expectations_str,
 149             self._port.test_platform_name(), is_debug_mode=False,
 150             is_lint_mode=False)
 151
 152         rs = run_webkit_tests.ResultSummary(expectations, test_paths)
 153         return test_paths, rs, expectations
 154
 155     def test_help_printer(self):
 156         # Here and below we'll call the "regular" printer err and the
 157         # buildbot printer out; this corresponds to how things run on the
 158         # bots with stderr and stdout.
 159         printer, err, out = self.get_printer()
 160
 161         # This routine should print something to stdout. testing what it is
 162         # is kind of pointless.
 163         printer.help_printing()
 164         self.assertFalse(err.empty())
 165         self.assertTrue(out.empty())
 166
 167     def do_switch_tests(self, method_name, switch, to_buildbot,
 168                         message='hello', exp_err=None, exp_bot=None):
 169         def do_helper(method_name, switch, message, exp_err, exp_bot):
 170             printer, err, bot = self.get_printer(['--print', switch])
 171             getattr(printer, method_name)(message)
 172             self.assertEqual(err.get(), exp_err)
 173             self.assertEqual(bot.get(), exp_bot)
 174
 175         if to_buildbot:
 176             if exp_err is None:
 177                 exp_err = []
 178             if exp_bot is None:
 179                 exp_bot = [message + "\n"]
 180         else:
 181             if exp_err is None:
 182                 exp_err = [message + "\n"]
 183             if exp_bot is None:
 184                 exp_bot = []
 185         do_helper(method_name, 'nothing', 'hello', [], [])
 186         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
 187         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
 188
 189     def test_configure_and_cleanup(self):
 190         # This test verifies that calling cleanup repeatedly and deleting
 191         # the object is safe.
 192         printer, err, out = self.get_printer(['--print', 'everything'])
 193         printer.cleanup()
 194         printer.cleanup()
 195         printer = None
 196
 197     def test_print_actual(self):
 198         # Actual results need to be logged to the buildbot's stream.
 199         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
 200
 201     def test_print_actual_buildbot(self):
 202         # FIXME: Test that the format of the actual results matches what the
 203         # buildbot is expecting.
 204         pass
 205
 206     def test_print_config(self):
 207         self.do_switch_tests('print_config', 'config', to_buildbot=False)
 208
 209     def test_print_expected(self):
 210         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
 211
 212     def test_print_timing(self):
 213         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
 214
 215     def test_print_update(self):
 216         # Note that there shouldn't be a carriage return here; updates()
 217         # are meant to be overwritten.
 218         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
 219                              message='hello', exp_err=['hello'])
 220
 221     def test_print_one_line_summary(self):
 222         printer, err, out = self.get_printer(['--print', 'nothing'])
 223         printer.print_one_line_summary(1, 1, 0)
 224         self.assertTrue(err.empty())
 225
 226         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
 227         printer.print_one_line_summary(1, 1, 0)
 228         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
 229
 230         printer, err, out = self.get_printer(['--print', 'everything'])
 231         printer.print_one_line_summary(1, 1, 0)
 232         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
 233
 234         err.reset()
 235         printer.print_one_line_summary(2, 1, 1)
 236         self.assertEquals(err.get(),
 237                           ["1 test ran as expected, 1 didn't:\n", "\n"])
 238
 239         err.reset()
 240         printer.print_one_line_summary(3, 2, 1)
 241         self.assertEquals(err.get(),
 242                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
 243
 244         err.reset()
 245         printer.print_one_line_summary(3, 2, 0)
 246         self.assertEquals(err.get(),
 247                           ['\n', "2 tests ran as expected (1 didn't run).\n",
 248                            '\n'])
 249
 250
 251     def test_print_test_result(self):
 252         # Note here that we don't use meaningful exp_str and got_str values;
 253         # the actual contents of the string are treated opaquely by
 254         # print_test_result() when tracing, and usually we don't want
 255         # to test what exactly is printed, just that something
 256         # was printed (or that nothing was printed).
 257         #
 258         # FIXME: this is actually some goofy layering; it would be nice
 259         # we could refactor it so that the args weren't redundant. Maybe
 260         # the TestResult should contain what was expected, and the
 261         # strings could be derived from the TestResult?
 262         printer, err, out = self.get_printer(['--print', 'nothing'])
 263         result = self.get_result('passes/image.html')
 264         printer.print_test_result(result, expected=False, exp_str='',
 265                                   got_str='')
 266         self.assertTrue(err.empty())
 267
 268         printer, err, out = self.get_printer(['--print', 'unexpected'])
 269         printer.print_test_result(result, expected=True, exp_str='',
 270                                   got_str='')
 271         self.assertTrue(err.empty())
 272         printer.print_test_result(result, expected=False, exp_str='',
 273                                   got_str='')
 274         self.assertEquals(err.get(),
 275                           ['  passes/image.html -> unexpected pass\n'])
 276
 277         printer, err, out = self.get_printer(['--print', 'everything'])
 278         printer.print_test_result(result, expected=True, exp_str='',
 279                                   got_str='')
 280         self.assertTrue(err.empty())
 281
 282         printer.print_test_result(result, expected=False, exp_str='',
 283                                   got_str='')
 284         self.assertEquals(err.get(),
 285                           ['  passes/image.html -> unexpected pass\n'])
 286
 287         printer, err, out = self.get_printer(['--print', 'nothing'])
 288         printer.print_test_result(result, expected=False, exp_str='',
 289                                   got_str='')
 290         self.assertTrue(err.empty())
 291
 292         printer, err, out = self.get_printer(['--print',
 293                                               'trace-unexpected'])
 294         printer.print_test_result(result, expected=True, exp_str='',
 295                                   got_str='')
 296         self.assertTrue(err.empty())
 297
 298         printer, err, out = self.get_printer(['--print',
 299                                               'trace-unexpected'])
 300         printer.print_test_result(result, expected=False, exp_str='',
 301                                   got_str='')
 302         self.assertFalse(err.empty())
 303
 304         printer, err, out = self.get_printer(['--print',
 305                                               'trace-unexpected'])
 306         result = self.get_result("passes/text.html")
 307         printer.print_test_result(result, expected=False, exp_str='',
 308                                   got_str='')
 309         self.assertFalse(err.empty())
 310
 311         err.reset()
 312         printer.print_test_result(result, expected=False, exp_str='',
 313                                   got_str='')
 314         self.assertFalse(err.empty())
 315
 316         printer, err, out = self.get_printer(['--print', 'trace-everything'])
 317         result = self.get_result('passes/image.html')
 318         printer.print_test_result(result, expected=True, exp_str='',
 319                                   got_str='')
 320         result = self.get_result('failures/expected/missing_text.html')
 321         printer.print_test_result(result, expected=True, exp_str='',
 322                                   got_str='')
 323         result = self.get_result('failures/expected/missing_check.html')
 324         printer.print_test_result(result, expected=True, exp_str='',
 325                                   got_str='')
 326         result = self.get_result('failures/expected/missing_image.html')
 327         printer.print_test_result(result, expected=True, exp_str='',
 328                                   got_str='')
 329         self.assertFalse(err.empty())
 330
 331         err.reset()
 332         printer.print_test_result(result, expected=False, exp_str='',
 333                                   got_str='')
 334
 335     def test_print_progress(self):
 336         expectations = ''
 337
 338         # test that we print nothing
 339         printer, err, out = self.get_printer(['--print', 'nothing'])
 340         tests = ['passes/text.html', 'failures/expected/timeout.html',
 341                  'failures/expected/crash.html']
 342         paths, rs, exp = self.get_result_summary(tests, expectations)
 343
 344         printer.print_progress(rs, False, paths)
 345         self.assertTrue(out.empty())
 346         self.assertTrue(err.empty())
 347
 348         printer.print_progress(rs, True, paths)
 349         self.assertTrue(out.empty())
 350         self.assertTrue(err.empty())
 351
 352         # test regular functionality
 353         printer, err, out = self.get_printer(['--print',
 354                                               'one-line-progress'])
 355         printer.print_progress(rs, False, paths)
 356         self.assertTrue(out.empty())
 357         self.assertFalse(err.empty())
 358
 359         err.reset()
 360         out.reset()
 361         printer.print_progress(rs, True, paths)
 362         self.assertFalse(err.empty())
 363         self.assertTrue(out.empty())
 364
 365     def test_print_progress__detailed(self):
 366         tests = ['passes/text.html', 'failures/expected/timeout.html',
 367                  'failures/expected/crash.html']
 368         expectations = 'failures/expected/timeout.html = TIMEOUT'
 369
 370         # first, test that it is disabled properly
 371         # should still print one-line-progress
 372         printer, err, out = self.get_printer(
 373             ['--print', 'detailed-progress'], single_threaded=False)
 374         paths, rs, exp = self.get_result_summary(tests, expectations)
 375         printer.print_progress(rs, False, paths)
 376         self.assertFalse(err.empty())
 377         self.assertTrue(out.empty())
 378
 379         # now test the enabled paths
 380         printer, err, out = self.get_printer(
 381             ['--print', 'detailed-progress'], single_threaded=True)
 382         paths, rs, exp = self.get_result_summary(tests, expectations)
 383         printer.print_progress(rs, False, paths)
 384         self.assertFalse(err.empty())
 385         self.assertTrue(out.empty())
 386
 387         err.reset()
 388         out.reset()
 389         printer.print_progress(rs, True, paths)
 390         self.assertFalse(err.empty())
 391         self.assertTrue(out.empty())
 392
 393         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
 394         rs.add(self.get_result('failures/expected/timeout.html'), True)
 395         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
 396         err.reset()
 397         out.reset()
 398         printer.print_progress(rs, False, paths)
 399         self.assertFalse(err.empty())
 400         self.assertTrue(out.empty())
 401
 402         # We only clear the meter when retrying w/ detailed-progress.
 403         err.reset()
 404         out.reset()
 405         printer.print_progress(rs, True, paths)
 406         self.assertFalse(err.empty())
 407         self.assertTrue(out.empty())
 408
 409         printer, err, out = self.get_printer(
 410             ['--print', 'detailed-progress,unexpected'], single_threaded=True)
 411         paths, rs, exp = self.get_result_summary(tests, expectations)
 412         printer.print_progress(rs, False, paths)
 413         self.assertFalse(err.empty())
 414         self.assertTrue(out.empty())
 415
 416         err.reset()
 417         out.reset()
 418         printer.print_progress(rs, True, paths)
 419         self.assertFalse(err.empty())
 420         self.assertTrue(out.empty())
 421
 422         rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
 423         rs.add(self.get_result('failures/expected/timeout.html'), True)
 424         rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
 425         err.reset()
 426         out.reset()
 427         printer.print_progress(rs, False, paths)
 428         self.assertFalse(err.empty())
 429         self.assertTrue(out.empty())
 430
 431         # We only clear the meter when retrying w/ detailed-progress.
 432         err.reset()
 433         out.reset()
 434         printer.print_progress(rs, True, paths)
 435         self.assertFalse(err.empty())
 436         self.assertTrue(out.empty())
 437
 438     def test_write_nothing(self):
 439         printer, err, out = self.get_printer(['--print', 'nothing'])
 440         printer.write("foo")
 441         self.assertTrue(err.empty())
 442
 443     def test_write_misc(self):
 444         printer, err, out = self.get_printer(['--print', 'misc'])
 445         printer.write("foo")
 446         self.assertFalse(err.empty())
 447         err.reset()
 448         printer.write("foo", "config")
 449         self.assertTrue(err.empty())
 450
 451     def test_write_everything(self):
 452         printer, err, out = self.get_printer(['--print', 'everything'])
 453         printer.write("foo")
 454         self.assertFalse(err.empty())
 455         err.reset()
 456         printer.write("foo", "config")
 457         self.assertFalse(err.empty())
 458
 459     def test_write_verbose(self):
 460         printer, err, out = self.get_printer(['--verbose'])
 461         printer.write("foo")
 462         self.assertTrue(not err.empty() and "foo" in err.get()[0])
 463         self.assertTrue(out.empty())
 464
 465     def test_print_unexpected_results(self):
 466         # This routine is the only one that prints stuff that the bots
 467         # care about.
 468         #
 469         # FIXME: there's some weird layering going on here. It seems
 470         # like we shouldn't be both using an expectations string and
 471         # having to specify whether or not the result was expected.
 472         # This whole set of tests should probably be rewritten.
 473         #
 474         # FIXME: Plus, the fact that we're having to call into
 475         # run_webkit_tests is clearly a layering inversion.
 476         def get_unexpected_results(expected, passing, flaky):
 477             """Return an unexpected results summary matching the input description.
 478
 479             There are a lot of different combinations of test results that
 480             can be tested; this routine produces various combinations based
 481             on the values of the input flags.
 482
 483             Args
 484                 expected: whether the tests ran as expected
 485                 passing: whether the tests should all pass
 486                 flaky: whether the tests should be flaky (if False, they
 487                     produce the same results on both runs; if True, they
 488                     all pass on the second run).
 489
 490             """
 491             paths, rs, exp = self.get_result_summary(tests, expectations)
 492             if expected:
 493                 rs.add(self.get_result('passes/text.html', test_expectations.PASS),
 494                        expected)
 495                 rs.add(self.get_result('failures/expected/timeout.html',
 496                        test_expectations.TIMEOUT), expected)
 497                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
 498                    expected)
 499             elif passing:
 500                 rs.add(self.get_result('passes/text.html'), expected)
 501                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
 502                 rs.add(self.get_result('failures/expected/crash.html'), expected)
 503             else:
 504                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
 505                        expected)
 506                 rs.add(self.get_result('failures/expected/timeout.html',
 507                        test_expectations.CRASH), expected)
 508                 rs.add(self.get_result('failures/expected/crash.html',
 509                                   test_expectations.TIMEOUT),
 510                    expected)
 511             retry = rs
 512             if flaky:
 513                 paths, retry, exp = self.get_result_summary(tests,
 514                                                 expectations)
 515                 retry.add(self.get_result('passes/text.html'), True)
 516                 retry.add(self.get_result('failures/expected/timeout.html'), True)
 517                 retry.add(self.get_result('failures/expected/crash.html'), True)
 518             unexpected_results = run_webkit_tests.summarize_unexpected_results(
 519                 self._port, exp, rs, retry)
 520             return unexpected_results
 521
 522         tests = ['passes/text.html', 'failures/expected/timeout.html',
 523                  'failures/expected/crash.html']
 524         expectations = ''
 525
 526         printer, err, out = self.get_printer(['--print', 'nothing'])
 527         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 528         printer.print_unexpected_results(ur)
 529         self.assertTrue(err.empty())
 530         self.assertTrue(out.empty())
 531
 532         printer, err, out = self.get_printer(['--print',
 533                                               'unexpected-results'])
 534
 535         # test everything running as expected
 536         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
 537         printer.print_unexpected_results(ur)
 538         self.assertTrue(err.empty())
 539         self.assertTrue(out.empty())
 540
 541         # test failures
 542         err.reset()
 543         out.reset()
 544         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 545         printer.print_unexpected_results(ur)
 546         self.assertTrue(err.empty())
 547         self.assertFalse(out.empty())
 548
 549         # test unexpected flaky results
 550         err.reset()
 551         out.reset()
 552         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
 553         printer.print_unexpected_results(ur)
 554         self.assertTrue(err.empty())
 555         self.assertFalse(out.empty())
 556
 557         # test unexpected passes
 558         err.reset()
 559         out.reset()
 560         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
 561         printer.print_unexpected_results(ur)
 562         self.assertTrue(err.empty())
 563         self.assertFalse(out.empty())
 564
 565         err.reset()
 566         out.reset()
 567         printer, err, out = self.get_printer(['--print', 'everything'])
 568         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 569         printer.print_unexpected_results(ur)
 570         self.assertTrue(err.empty())
 571         self.assertFalse(out.empty())
 572
 573         expectations = """
 574 failures/expected/crash.html = CRASH
 575 failures/expected/timeout.html = TIMEOUT
 576 """
 577         err.reset()
 578         out.reset()
 579         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 580         printer.print_unexpected_results(ur)
 581         self.assertTrue(err.empty())
 582         self.assertFalse(out.empty())
 583
 584         err.reset()
 585         out.reset()
 586         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
 587         printer.print_unexpected_results(ur)
 588         self.assertTrue(err.empty())
 589         self.assertFalse(out.empty())
 590
 591         # Test handling of --verbose as well.
 592         err.reset()
 593         out.reset()
 594         printer, err, out = self.get_printer(['--verbose'])
 595         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 596         printer.print_unexpected_results(ur)
 597         self.assertTrue(err.empty())
 598         self.assertFalse(out.empty())
 599
 600     def test_print_unexpected_results_buildbot(self):
 601         # FIXME: Test that print_unexpected_results() produces the printer the
 602         # buildbot is expecting.
 603         pass
 604
 605 if __name__ == '__main__':
 606     unittest.main()