Web Inspector: chromium: I'd like to add a script for running perf tests for WebInspe...
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner.py
1 #!/usr/bin/env python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Run Inspector's perf tests in perf mode."""
31
32 import logging
33 import optparse
34 import re
35 import sys
36
37 from webkitpy.common import find_files
38 from webkitpy.common.host import Host
39 from webkitpy.layout_tests.port.driver import DriverInput
40 from webkitpy.layout_tests.views import printing
41
42 _log = logging.getLogger(__name__)
43
44
45 class PerfTestsRunner(object):
46     _perf_tests_base_dir = 'PerformanceTests'
47     _result_regex = re.compile('^RESULT .*$')
48
49     def __init__(self, perf_tests_dir, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None):
50         self._perf_tests_dir = perf_tests_dir
51         self._buildbot_output = buildbot_output
52         self._options, self._args = self._parse_args(args)
53         self._host = Host()
54         self._host._initialize_scm()
55         self._port = self._host.port_factory.get(self._options.platform, self._options)
56         self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
57         self._webkit_base_dir_len = len(self._port.config.webkit_base_dir())
58
59     def _parse_args(self, args=None):
60         print_options = printing.print_options()
61
62         perf_option_list = [
63             optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
64                                  help='Set the configuration to Debug'),
65             optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
66                                  help='Set the configuration to Release'),
67             optparse.make_option("--platform",
68                                  help="Specify port/platform being tested (i.e. chromium-mac)"),
69             optparse.make_option("--build-directory",
70                                  help="Path to the directory under which build files are kept (should not include configuration)"),
71             optparse.make_option("--time-out-ms", default=30000,
72                                  help="Set the timeout for each test"),
73             ]
74
75         option_list = (perf_option_list + print_options)
76         return optparse.OptionParser(option_list=option_list).parse_args(args)
77
78     def _collect_tests(self, webkit_base, filesystem=None):
79         """Return the list of tests found."""
80
81         def _is_test_file(filesystem, dirname, filename):
82             return filename.endswith('.html')
83
84         filesystem = filesystem or self._host.filesystem
85         base_dir = filesystem.join(webkit_base, self._perf_tests_base_dir, self._perf_tests_dir)
86         return find_files.find(filesystem, base_dir, paths=self._args, file_filter=_is_test_file)
87
88     def run(self):
89         if self._options.help_printing:
90             self._printer.help_printing()
91             self._printer.cleanup()
92             return 0
93
94         if not self._port.check_build(needs_http=False):
95             _log.error("Build not up to date for %s" % self._port._path_to_driver())
96             return -1
97
98         # We wrap any parts of the run that are slow or likely to raise exceptions
99         # in a try/finally to ensure that we clean up the logging configuration.
100         unexpected = -1
101         try:
102             tests = self._collect_tests(self._port.webkit_base())
103             unexpected = self._run_tests_set(tests, self._port)
104         finally:
105             self._printer.cleanup()
106
107         return unexpected
108
109     def _run_tests_set(self, tests, port):
110         result_count = len(tests)
111         expected = 0
112         unexpected = 0
113         self._printer.print_one_line_summary(result_count, 0, 0)
114         driver_need_restart = False
115         driver = None
116
117         for test in tests:
118             if driver_need_restart:
119                 _log.debug("%s killing driver" % test)
120                 driver.stop()
121                 driver = None
122             if not driver:
123                 driver = port.create_driver(worker_number=1)
124
125             test_failed, driver_need_restart = self._run_single_test(test, driver)
126             if test_failed:
127                 unexpected = unexpected + 1
128             else:
129                 expected = expected + 1
130
131             self._printer.print_one_line_summary(result_count, expected, unexpected)
132
133         if driver:
134             driver.stop()
135
136         return unexpected
137
138     def _run_single_test(self, test, driver):
139         test_failed = False
140         driver_need_restart = False
141         output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
142
143         if output.text == None:
144             test_failed = True
145         elif output.timeout:
146             self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
147             test_failed = True
148             driver_need_restart = True
149         elif output.crash:
150             self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
151             driver_need_restart = True
152             test_failed = True
153         else:
154             got_a_result = False
155             for line in re.split('\n', output.text):
156                 if self._result_regex.match(line):
157                     self._buildbot_output.write("%s\n" % line)
158                     got_a_result = True
159                 elif not len(line) == 0:
160                     test_failed = True
161                     self._printer.write("%s" % line)
162             test_failed = test_failed or not got_a_result
163
164         if len(output.error):
165             self._printer.write('error:\n%s' % output.error)
166             test_failed = True
167
168         return test_failed, driver_need_restart