Unreviewed build fix after r105256.
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner.py
1 #!/usr/bin/env python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Run Inspector's perf tests in perf mode."""
31
32 import json
33 import logging
34 import optparse
35 import re
36 import sys
37 import time
38
39 from webkitpy.common import find_files
40 from webkitpy.common.host import Host
41 from webkitpy.layout_tests.port.driver import DriverInput
42 from webkitpy.layout_tests.views import printing
43
44 _log = logging.getLogger(__name__)
45
46
47 class PerfTestsRunner(object):
48     _perf_tests_base_dir = 'PerformanceTests'
49     _test_directories_for_chromium_style_tests = ['inspector']
50
51     def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None):
52         self._buildbot_output = buildbot_output
53         self._options, self._args = PerfTestsRunner._parse_args(args)
54         if port:
55             self._port = port
56             self._host = self._port.host
57         else:
58             self._host = Host()
59             self._port = self._host.port_factory.get(self._options.platform, self._options)
60         self._host._initialize_scm()
61         self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
62         self._webkit_base_dir_len = len(self._port.webkit_base())
63         self._base_path = self._host.filesystem.join(self._port.webkit_base(), self._perf_tests_base_dir)
64         self._results = {}
65         self._timestamp = time.time()
66
67     @staticmethod
68     def _parse_args(args=None):
69         print_options = printing.print_options()
70
71         perf_option_list = [
72             optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
73                                  help='Set the configuration to Debug'),
74             optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
75                                  help='Set the configuration to Release'),
76             optparse.make_option("--platform",
77                                  help="Specify port/platform being tested (i.e. chromium-mac)"),
78             optparse.make_option("--build", dest="build", action="store_true", default=True,
79                                 help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
80             optparse.make_option("--build-directory",
81                                  help="Path to the directory under which build files are kept (should not include configuration)"),
82             optparse.make_option("--time-out-ms", default=30000,
83                                  help="Set the timeout for each test"),
84             optparse.make_option("--output-json-path",
85                                  help="Filename of the JSON file that summaries the results"),
86             optparse.make_option("--source-json-path",
87                                  help="Path to a JSON file to be merged into the JSON file when --output-json-path is specified"),
88             ]
89
90         option_list = (perf_option_list + print_options)
91         return optparse.OptionParser(option_list=option_list).parse_args(args)
92
93     def _collect_tests(self):
94         """Return the list of tests found."""
95
96         def _is_test_file(filesystem, dirname, filename):
97             return filename.endswith('.html')
98
99         return find_files.find(self._host.filesystem, self._base_path, paths=self._args, file_filter=_is_test_file)
100
101     def run(self):
102         if self._options.help_printing:
103             self._printer.help_printing()
104             self._printer.cleanup()
105             return 0
106
107         if not self._port.check_build(needs_http=False):
108             _log.error("Build not up to date for %s" % self._port._path_to_driver())
109             return -1
110
111         # We wrap any parts of the run that are slow or likely to raise exceptions
112         # in a try/finally to ensure that we clean up the logging configuration.
113         unexpected = -1
114         try:
115             tests = self._collect_tests()
116             unexpected = self._run_tests_set(tests, self._port)
117         finally:
118             self._printer.cleanup()
119
120         if not self._generate_json_if_specified(self._timestamp) and not unexpected:
121             return -2
122
123         return unexpected
124
125     def _generate_json_if_specified(self, timestamp):
126         output_json_path = self._options.output_json_path
127         if not output_json_path:
128             return True
129
130         revision = self._host.scm().head_svn_revision()
131         contents = {'timestamp': int(timestamp), 'revision': revision, 'results': self._results}
132
133         filesystem = self._host.filesystem
134         source_json_path = self._options.source_json_path
135         if source_json_path:
136             try:
137                 source_json_file = filesystem.open_text_file_for_reading(source_json_path)
138                 source_json = json.load(source_json_file)
139             except:
140                 _log.error("Failed to parse %s" % source_json_path)
141                 return False
142             if not isinstance(source_json, dict):
143                 _log.error("The source JSON was not a dictionary")
144                 return False
145             contents = dict(source_json.items() + contents.items())
146
147         filesystem.write_text_file(output_json_path, json.dumps(contents))
148         return True
149
150     def _print_status(self, tests, expected, unexpected):
151         if len(tests) == expected + unexpected:
152             status = "Ran %d tests" % len(tests)
153         else:
154             status = "Running %d of %d tests" % (expected + unexpected + 1, len(tests))
155         if unexpected:
156             status += " (%d didn't run)" % unexpected
157         self._printer.write(status)
158
159     def _run_tests_set(self, tests, port):
160         result_count = len(tests)
161         expected = 0
162         unexpected = 0
163         driver_need_restart = False
164         driver = None
165
166         for test in tests:
167             if driver_need_restart:
168                 _log.debug("%s killing driver" % test)
169                 driver.stop()
170                 driver = None
171             if not driver:
172                 driver = port.create_driver(worker_number=1)
173
174             relative_test_path = self._host.filesystem.relpath(test, self._base_path)
175             self._printer.write('Running %s (%d of %d)' % (relative_test_path, expected + unexpected + 1, len(tests)))
176
177             is_chromium_style = self._host.filesystem.split(relative_test_path)[0] in self._test_directories_for_chromium_style_tests
178             test_failed, driver_need_restart = self._run_single_test(test, driver, is_chromium_style)
179             if test_failed:
180                 unexpected = unexpected + 1
181             else:
182                 expected = expected + 1
183
184             self._printer.write('')
185
186         if driver:
187             driver.stop()
188
189         return unexpected
190
191     _inspector_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
192
193     def _process_chromium_style_test_result(self, test, output):
194         test_failed = False
195         got_a_result = False
196         for line in re.split('\n', output.text):
197             resultLine = self._inspector_result_regex.match(line)
198             if resultLine:
199                 self._results[resultLine.group('name').replace(' ', '')] = int(resultLine.group('value'))
200                 self._buildbot_output.write("%s\n" % line)
201                 got_a_result = True
202             elif not len(line) == 0:
203                 test_failed = True
204                 self._printer.write("%s" % line)
205         return test_failed or not got_a_result
206
207     _lines_to_ignore_in_parser_result = [
208         re.compile(r'^Running \d+ times$'),
209         re.compile(r'^Ignoring warm-up '),
210         re.compile(r'^\d+$'),
211     ]
212
213     def _should_ignore_line_in_parser_test_result(self, line):
214         if not line:
215             return True
216         for regex in self._lines_to_ignore_in_parser_result:
217             if regex.match(line):
218                 return True
219         return False
220
221     def _process_parser_test_result(self, test, output):
222         got_a_result = False
223         test_failed = False
224         filesystem = self._host.filesystem
225         category, test_name = filesystem.split(filesystem.relpath(test, self._base_path))
226         test_name = filesystem.splitext(test_name)[0]
227         results = {}
228         keys = ['avg', 'median', 'stdev', 'min', 'max']
229         score_regex = re.compile(r'^(' + r'|'.join(keys) + r')\s+([0-9\.]+)')
230         for line in re.split('\n', output.text):
231             score = score_regex.match(line)
232             if score:
233                 results[score.group(1)] = score.group(2)
234                 continue
235
236             if not self._should_ignore_line_in_parser_test_result(line):
237                 test_failed = True
238                 self._printer.write("%s" % line)
239
240         if test_failed or set(keys) != set(results.keys()):
241             return True
242         self._results[test_name] = results
243         self._buildbot_output.write('RESULT %s: %s= %s ms\n' % (category, test_name, results['avg']))
244         self._buildbot_output.write(', '.join(['%s= %s ms' % (key, results[key]) for key in keys[1:]]) + '\n')
245         return False
246
247     def _run_single_test(self, test, driver, is_chromium_style):
248         test_failed = False
249         driver_need_restart = False
250         output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
251
252         if output.text == None:
253             test_failed = True
254         elif output.timeout:
255             self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
256             test_failed = True
257             driver_need_restart = True
258         elif output.crash:
259             self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
260             driver_need_restart = True
261             test_failed = True
262         else:
263             if is_chromium_style:
264                 test_failed = self._process_chromium_style_test_result(test, output)
265             else:
266                 test_failed = self._process_parser_test_result(test, output)
267
268         if len(output.error):
269             self._printer.write('error:\n%s' % output.error)
270             test_failed = True
271
272         if test_failed:
273             self._printer.write('FAILED')
274
275         return test_failed, driver_need_restart