Tools/Scripts/webkitpy/layout_tests/views/buildbot_results_unittest.py

   1 # Copyright (C) 2012 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import StringIO
  30 import unittest
  31
  32 from webkitpy.common.host_mock import MockHost
  33 from webkitpy.tool.mocktool import MockOptions
  34
  35 from webkitpy.layout_tests.models import test_expectations
  36 from webkitpy.layout_tests.models import test_failures
  37 from webkitpy.layout_tests.models import test_run_results
  38 from webkitpy.layout_tests.models import test_run_results
  39 from webkitpy.layout_tests.models import test_run_results_unittest
  40 from webkitpy.layout_tests.views import buildbot_results
  41
  42
  43 class BuildBotPrinterTests(unittest.TestCase):
  44     def assertEmpty(self, stream):
  45         self.assertFalse(stream.getvalue())
  46
  47     def assertNotEmpty(self, stream):
  48         self.assertTrue(stream.getvalue())
  49
  50     def get_printer(self):
  51         stream = StringIO.StringIO()
  52         printer = buildbot_results.BuildBotPrinter(stream, debug_logging=True)
  53         return printer, stream
  54
  55     def test_print_unexpected_results(self):
  56         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
  57         printer, out = self.get_printer()
  58
  59         # test everything running as expected
  60         DASHED_LINE = "-" * 78 + "\n"
  61         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
  62         printer.print_unexpected_results(summary)
  63         self.assertEqual(out.getvalue(), DASHED_LINE)
  64
  65         # test failures
  66         printer, out = self.get_printer()
  67         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
  68         printer.print_unexpected_results(summary)
  69         self.assertNotEmpty(out)
  70
  71         # test unexpected flaky
  72         printer, out = self.get_printer()
  73         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=True)
  74         printer.print_unexpected_results(summary)
  75         self.assertNotEmpty(out)
  76
  77         printer, out = self.get_printer()
  78         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
  79         printer.print_unexpected_results(summary)
  80         self.assertNotEmpty(out)
  81
  82         printer, out = self.get_printer()
  83         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=False, flaky=False)
  84         printer.print_unexpected_results(summary)
  85         self.assertNotEmpty(out)
  86
  87         printer, out = self.get_printer()
  88         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
  89         printer.print_unexpected_results(summary)
  90         self.assertNotEmpty(out)
  91
  92     def test_print_unexpected_results_with_options(self):
  93         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
  94
  95         DASHED_LINE = "-" * 78 + "\n"
  96
  97         port._options.pixel_tests = True
  98         port._options.world_leaks = False
  99         printer, out = self.get_printer()
 100         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
 101         printer.print_unexpected_results(summary)
 102         self.assertEqual(out.getvalue(), DASHED_LINE)
 103
 104         port._options.pixel_tests = False
 105         port._options.world_leaks = True
 106         printer, out = self.get_printer()
 107         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
 108         printer.print_unexpected_results(summary)
 109         self.assertEqual(out.getvalue(), DASHED_LINE)
 110
 111         port._options.pixel_tests = True
 112         port._options.world_leaks = True
 113         printer, out = self.get_printer()
 114         summary = test_run_results_unittest.summarized_results(port, expected=True, passing=False, flaky=False)
 115         printer.print_unexpected_results(summary)
 116         self.assertEqual(out.getvalue(), DASHED_LINE)
 117
 118     def test_print_results(self):
 119         port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
 120         printer, out = self.get_printer()
 121         initial_results = test_run_results_unittest.run_results(port)
 122         summary = test_run_results_unittest.summarized_results(port, expected=False, passing=True, flaky=False)
 123         details = test_run_results.RunDetails(summary['num_regressions'], summary, initial_results, None)
 124         printer.print_results(details)
 125         self.assertNotEmpty(out)