Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2011 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Run Inspector's perf tests in perf mode."""
  31
  32 import logging
  33 import optparse
  34 import re
  35 import sys
  36
  37 from webkitpy.common import find_files
  38 from webkitpy.common.host import Host
  39 from webkitpy.layout_tests.port.driver import DriverInput
  40 from webkitpy.layout_tests.views import printing
  41
  42 _log = logging.getLogger(__name__)
  43
  44
  45 class PerfTestsRunner(object):
  46     _perf_tests_base_dir = 'PerformanceTests'
  47     _result_regex = re.compile('^RESULT .*$')
  48
  49     def __init__(self, perf_tests_dir, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None):
  50         self._perf_tests_dir = perf_tests_dir
  51         self._buildbot_output = buildbot_output
  52         self._options, self._args = self._parse_args(args)
  53         self._host = Host()
  54         self._host._initialize_scm()
  55         self._port = self._host.port_factory.get(self._options.platform, self._options)
  56         self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
  57         self._webkit_base_dir_len = len(self._port.config.webkit_base_dir())
  58
  59     def _parse_args(self, args=None):
  60         print_options = printing.print_options()
  61
  62         perf_option_list = [
  63             optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
  64                                  help='Set the configuration to Debug'),
  65             optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
  66                                  help='Set the configuration to Release'),
  67             optparse.make_option("--platform",
  68                                  help="Specify port/platform being tested (i.e. chromium-mac)"),
  69             optparse.make_option("--build-directory",
  70                                  help="Path to the directory under which build files are kept (should not include configuration)"),
  71             optparse.make_option("--time-out-ms", default=30000,
  72                                  help="Set the timeout for each test"),
  73             ]
  74
  75         option_list = (perf_option_list + print_options)
  76         return optparse.OptionParser(option_list=option_list).parse_args(args)
  77
  78     def _collect_tests(self, webkit_base, filesystem=None):
  79         """Return the list of tests found."""
  80
  81         def _is_test_file(filesystem, dirname, filename):
  82             return filename.endswith('.html')
  83
  84         filesystem = filesystem or self._host.filesystem
  85         base_dir = filesystem.join(webkit_base, self._perf_tests_base_dir, self._perf_tests_dir)
  86         return find_files.find(filesystem, base_dir, paths=self._args, file_filter=_is_test_file)
  87
  88     def run(self):
  89         if self._options.help_printing:
  90             self._printer.help_printing()
  91             self._printer.cleanup()
  92             return 0
  93
  94         if not self._port.check_build(needs_http=False):
  95             _log.error("Build not up to date for %s" % self._port._path_to_driver())
  96             return -1
  97
  98         # We wrap any parts of the run that are slow or likely to raise exceptions
  99         # in a try/finally to ensure that we clean up the logging configuration.
 100         unexpected = -1
 101         try:
 102             tests = self._collect_tests(self._port.webkit_base())
 103             unexpected = self._run_tests_set(tests, self._port)
 104         finally:
 105             self._printer.cleanup()
 106
 107         return unexpected
 108
 109     def _run_tests_set(self, tests, port):
 110         result_count = len(tests)
 111         expected = 0
 112         unexpected = 0
 113         self._printer.print_one_line_summary(result_count, 0, 0)
 114         driver_need_restart = False
 115         driver = None
 116
 117         for test in tests:
 118             if driver_need_restart:
 119                 _log.debug("%s killing driver" % test)
 120                 driver.stop()
 121                 driver = None
 122             if not driver:
 123                 driver = port.create_driver(worker_number=1)
 124
 125             test_failed, driver_need_restart = self._run_single_test(test, driver)
 126             if test_failed:
 127                 unexpected = unexpected + 1
 128             else:
 129                 expected = expected + 1
 130
 131             self._printer.print_one_line_summary(result_count, expected, unexpected)
 132
 133         if driver:
 134             driver.stop()
 135
 136         return unexpected
 137
 138     def _run_single_test(self, test, driver):
 139         test_failed = False
 140         driver_need_restart = False
 141         output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
 142
 143         if output.text == None:
 144             test_failed = True
 145         elif output.timeout:
 146             self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
 147             test_failed = True
 148             driver_need_restart = True
 149         elif output.crash:
 150             self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
 151             driver_need_restart = True
 152             test_failed = True
 153         else:
 154             got_a_result = False
 155             for line in re.split('\n', output.text):
 156                 if self._result_regex.match(line):
 157                     self._buildbot_output.write("%s\n" % line)
 158                     got_a_result = True
 159                 elif not len(line) == 0:
 160                     test_failed = True
 161                     self._printer.write("%s" % line)
 162             test_failed = test_failed or not got_a_result
 163
 164         if len(output.error):
 165             self._printer.write('error:\n%s' % output.error)
 166             test_failed = True
 167
 168         return test_failed, driver_need_restart