Some perf tests time out when ran by run-perf-tests
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner.py
1 #!/usr/bin/env python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Run Inspector's perf tests in perf mode."""
31
32 import json
33 import logging
34 import optparse
35 import re
36 import sys
37 import time
38
39 from webkitpy.common import find_files
40 from webkitpy.common.host import Host
41 from webkitpy.layout_tests.port.driver import DriverInput
42 from webkitpy.layout_tests.views import printing
43
44 _log = logging.getLogger(__name__)
45
46
47 class PerfTestsRunner(object):
48     _perf_tests_base_dir = 'PerformanceTests'
49     _test_directories_for_chromium_style_tests = ['inspector']
50
51     def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None):
52         self._buildbot_output = buildbot_output
53         self._options, self._args = PerfTestsRunner._parse_args(args)
54         if port:
55             self._port = port
56             self._host = self._port.host
57         else:
58             self._host = Host()
59             self._port = self._host.port_factory.get(self._options.platform, self._options)
60         self._host._initialize_scm()
61         self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
62         self._webkit_base_dir_len = len(self._port.webkit_base())
63         self._base_path = self._host.filesystem.join(self._port.webkit_base(), self._perf_tests_base_dir)
64         self._results = {}
65         self._timestamp = time.time()
66
67     @staticmethod
68     def _parse_args(args=None):
69         print_options = printing.print_options()
70
71         perf_option_list = [
72             optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
73                                  help='Set the configuration to Debug'),
74             optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
75                                  help='Set the configuration to Release'),
76             optparse.make_option("--platform",
77                                  help="Specify port/platform being tested (i.e. chromium-mac)"),
78             optparse.make_option("--build", dest="build", action="store_true", default=True,
79                                 help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
80             optparse.make_option("--build-directory",
81                                  help="Path to the directory under which build files are kept (should not include configuration)"),
82             optparse.make_option("--time-out-ms", default=600 * 1000,
83                                  help="Set the timeout for each test"),
84             optparse.make_option("--output-json-path",
85                                  help="Filename of the JSON file that summaries the results"),
86             optparse.make_option("--source-json-path",
87                                  help="Path to a JSON file to be merged into the JSON file when --output-json-path is specified"),
88             ]
89
90         option_list = (perf_option_list + print_options)
91         return optparse.OptionParser(option_list=option_list).parse_args(args)
92
93     def _collect_tests(self):
94         """Return the list of tests found."""
95
96         def _is_test_file(filesystem, dirname, filename):
97             return filename.endswith('.html')
98
99         skipped_directories = set(['.svn', 'resources'])
100         tests = find_files.find(self._host.filesystem, self._base_path, self._args, skipped_directories, _is_test_file)
101         return [test for test in tests if not self._port.skips_perf_test(self._port.relative_perf_test_filename(test))]
102
103     def run(self):
104         if self._options.help_printing:
105             self._printer.help_printing()
106             self._printer.cleanup()
107             return 0
108
109         if not self._port.check_build(needs_http=False):
110             _log.error("Build not up to date for %s" % self._port._path_to_driver())
111             return -1
112
113         # We wrap any parts of the run that are slow or likely to raise exceptions
114         # in a try/finally to ensure that we clean up the logging configuration.
115         unexpected = -1
116         try:
117             tests = self._collect_tests()
118             unexpected = self._run_tests_set(sorted(list(tests)), self._port)
119         finally:
120             self._printer.cleanup()
121
122         if not self._generate_json_if_specified(self._timestamp) and not unexpected:
123             return -2
124
125         return unexpected
126
127     def _generate_json_if_specified(self, timestamp):
128         output_json_path = self._options.output_json_path
129         if not output_json_path:
130             return True
131
132         revision = self._host.scm().head_svn_revision()
133         contents = {'timestamp': int(timestamp), 'revision': revision, 'results': self._results}
134
135         filesystem = self._host.filesystem
136         source_json_path = self._options.source_json_path
137         if source_json_path:
138             try:
139                 source_json_file = filesystem.open_text_file_for_reading(source_json_path)
140                 source_json = json.load(source_json_file)
141             except:
142                 _log.error("Failed to parse %s" % source_json_path)
143                 return False
144             if not isinstance(source_json, dict):
145                 _log.error("The source JSON was not a dictionary")
146                 return False
147             contents = dict(source_json.items() + contents.items())
148
149         filesystem.write_text_file(output_json_path, json.dumps(contents))
150         return True
151
152     def _print_status(self, tests, expected, unexpected):
153         if len(tests) == expected + unexpected:
154             status = "Ran %d tests" % len(tests)
155         else:
156             status = "Running %d of %d tests" % (expected + unexpected + 1, len(tests))
157         if unexpected:
158             status += " (%d didn't run)" % unexpected
159         self._printer.write(status)
160
161     def _run_tests_set(self, tests, port):
162         result_count = len(tests)
163         expected = 0
164         unexpected = 0
165         driver_need_restart = False
166         driver = None
167
168         for test in tests:
169             if driver_need_restart:
170                 _log.debug("%s killing driver" % test)
171                 driver.stop()
172                 driver = None
173             if not driver:
174                 driver = port.create_driver(worker_number=1, no_timeout=True)
175
176             relative_test_path = self._host.filesystem.relpath(test, self._base_path)
177             self._printer.write('Running %s (%d of %d)' % (relative_test_path, expected + unexpected + 1, len(tests)))
178
179             is_chromium_style = self._host.filesystem.split(relative_test_path)[0] in self._test_directories_for_chromium_style_tests
180             test_failed, driver_need_restart = self._run_single_test(test, driver, is_chromium_style)
181             if test_failed:
182                 unexpected = unexpected + 1
183             else:
184                 expected = expected + 1
185
186             self._printer.write('')
187
188         if driver:
189             driver.stop()
190
191         return unexpected
192
193     _inspector_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
194
195     def _process_chromium_style_test_result(self, test, output):
196         test_failed = False
197         got_a_result = False
198         for line in re.split('\n', output.text):
199             resultLine = self._inspector_result_regex.match(line)
200             if resultLine:
201                 self._results[resultLine.group('name').replace(' ', '')] = int(resultLine.group('value'))
202                 self._buildbot_output.write("%s\n" % line)
203                 got_a_result = True
204             elif not len(line) == 0:
205                 test_failed = True
206                 self._printer.write("%s" % line)
207         return test_failed or not got_a_result
208
209     _lines_to_ignore_in_parser_result = [
210         re.compile(r'^Running \d+ times$'),
211         re.compile(r'^Ignoring warm-up '),
212         re.compile(r'^\d+$'),
213     ]
214
215     def _should_ignore_line_in_parser_test_result(self, line):
216         if not line:
217             return True
218         for regex in self._lines_to_ignore_in_parser_result:
219             if regex.match(line):
220                 return True
221         return False
222
223     def _process_parser_test_result(self, test, output):
224         got_a_result = False
225         test_failed = False
226         filesystem = self._host.filesystem
227         category, test_name = filesystem.split(filesystem.relpath(test, self._base_path))
228         test_name = filesystem.splitext(test_name)[0]
229         results = {}
230         keys = ['avg', 'median', 'stdev', 'min', 'max']
231         score_regex = re.compile(r'^(' + r'|'.join(keys) + r')\s+([0-9\.]+)')
232         for line in re.split('\n', output.text):
233             score = score_regex.match(line)
234             if score:
235                 results[score.group(1)] = score.group(2)
236                 continue
237
238             if not self._should_ignore_line_in_parser_test_result(line):
239                 test_failed = True
240                 self._printer.write("%s" % line)
241
242         if test_failed or set(keys) != set(results.keys()):
243             return True
244         self._results[test_name] = results
245         self._buildbot_output.write('RESULT %s: %s= %s ms\n' % (category, test_name, results['avg']))
246         self._buildbot_output.write(', '.join(['%s= %s ms' % (key, results[key]) for key in keys[1:]]) + '\n')
247         return False
248
249     def _run_single_test(self, test, driver, is_chromium_style):
250         test_failed = False
251         driver_need_restart = False
252         output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
253
254         if output.text == None:
255             test_failed = True
256         elif output.timeout:
257             self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
258             test_failed = True
259             driver_need_restart = True
260         elif output.crash:
261             self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
262             driver_need_restart = True
263             test_failed = True
264         else:
265             if is_chromium_style:
266                 test_failed = self._process_chromium_style_test_result(test, output)
267             else:
268                 test_failed = self._process_parser_test_result(test, output)
269
270         if len(output.error):
271             self._printer.write('error:\n%s' % output.error)
272             test_failed = True
273
274         if test_failed:
275             self._printer.write('FAILED')
276
277         return test_failed, driver_need_restart