ce1c7e90385324cb56ddd86091f481f637616d9f
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Run Inspector's perf tests in perf mode."""
31
32 import json
33 import logging
34 import optparse
35 import re
36 import sys
37 import time
38
39 from webkitpy.common import find_files
40 from webkitpy.common.host import Host
41 from webkitpy.common.net.file_uploader import FileUploader
42 from webkitpy.layout_tests.port.driver import DriverInput
43 from webkitpy.layout_tests.views import printing
44
45 _log = logging.getLogger(__name__)
46
47
48 class PerfTestsRunner(object):
49     _test_directories_for_chromium_style_tests = ['inspector']
50     _default_branch = 'webkit-trunk'
51     _EXIT_CODE_BAD_BUILD = -1
52     _EXIT_CODE_BAD_JSON = -2
53     _EXIT_CODE_FAILED_UPLOADING = -3
54
55     def __init__(self, regular_output=sys.stderr, buildbot_output=sys.stdout, args=None, port=None):
56         self._buildbot_output = buildbot_output
57         self._options, self._args = PerfTestsRunner._parse_args(args)
58         if port:
59             self._port = port
60             self._host = self._port.host
61         else:
62             self._host = Host()
63             self._port = self._host.port_factory.get(self._options.platform, self._options)
64         self._host._initialize_scm()
65         self._printer = printing.Printer(self._port, self._options, regular_output, buildbot_output, configure_logging=False)
66         self._webkit_base_dir_len = len(self._port.webkit_base())
67         self._base_path = self._port.perf_tests_dir()
68         self._results = {}
69         self._timestamp = time.time()
70
71     @staticmethod
72     def _parse_args(args=None):
73         print_options = printing.print_options()
74
75         perf_option_list = [
76             optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration",
77                 help='Set the configuration to Debug'),
78             optparse.make_option('--release', action='store_const', const='Release', dest="configuration",
79                 help='Set the configuration to Release'),
80             optparse.make_option("--platform",
81                 help="Specify port/platform being tested (i.e. chromium-mac)"),
82             optparse.make_option("--chromium",
83                 action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
84             optparse.make_option("--builder-name",
85                 help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")),
86             optparse.make_option("--build-number",
87                 help=("The build number of the builder running this script.")),
88             optparse.make_option("--build", dest="build", action="store_true", default=True,
89                 help="Check to ensure the DumpRenderTree build is up-to-date (default)."),
90             optparse.make_option("--build-directory",
91                 help="Path to the directory under which build files are kept (should not include configuration)"),
92             optparse.make_option("--time-out-ms", default=600 * 1000,
93                 help="Set the timeout for each test"),
94             optparse.make_option("--pause-before-testing", dest="pause_before_testing", action="store_true", default=False,
95                 help="Pause before running the tests to let user attach a performance monitor."),
96             optparse.make_option("--output-json-path",
97                 help="Filename of the JSON file that summaries the results"),
98             optparse.make_option("--source-json-path",
99                 help="Path to a JSON file to be merged into the JSON file when --output-json-path is present"),
100             optparse.make_option("--test-results-server",
101                 help="Upload the generated JSON file to the specified server when --output-json-path is present"),
102             optparse.make_option("--webkit-test-runner", "-2", action="store_true",
103                 help="Use WebKitTestRunner rather than DumpRenderTree."),
104             ]
105
106         option_list = (perf_option_list + print_options)
107         return optparse.OptionParser(option_list=option_list).parse_args(args)
108
109     def _collect_tests(self):
110         """Return the list of tests found."""
111
112         def _is_test_file(filesystem, dirname, filename):
113             return filename.endswith('.html')
114
115         paths = []
116         for arg in self._args:
117             paths.append(arg)
118             relpath = self._host.filesystem.relpath(arg, self._base_path)
119             if relpath:
120                 paths.append(relpath)
121
122         skipped_directories = set(['.svn', 'resources'])
123         tests = find_files.find(self._host.filesystem, self._base_path, paths, skipped_directories, _is_test_file)
124         return [test for test in tests if not self._port.skips_perf_test(self._port.relative_perf_test_filename(test))]
125
126     def run(self):
127         if self._options.help_printing:
128             self._printer.help_printing()
129             self._printer.cleanup()
130             return 0
131
132         if not self._port.check_build(needs_http=False):
133             _log.error("Build not up to date for %s" % self._port._path_to_driver())
134             return self._EXIT_CODE_BAD_BUILD
135
136         # We wrap any parts of the run that are slow or likely to raise exceptions
137         # in a try/finally to ensure that we clean up the logging configuration.
138         unexpected = -1
139         try:
140             tests = self._collect_tests()
141             unexpected = self._run_tests_set(sorted(list(tests)), self._port)
142         finally:
143             self._printer.cleanup()
144
145         options = self._options
146         if self._options.output_json_path:
147             # FIXME: Add --branch or auto-detect the branch we're in
148             test_results_server = options.test_results_server
149             branch = self._default_branch if test_results_server else None
150             build_number = int(options.build_number) if options.build_number else None
151             if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path,
152                 branch, options.platform, options.builder_name, build_number) and not unexpected:
153                 return self._EXIT_CODE_BAD_JSON
154             if test_results_server and not self._upload_json(test_results_server, options.output_json_path):
155                 return self._EXIT_CODE_FAILED_UPLOADING
156
157         return unexpected
158
159     def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number):
160         contents = {'timestamp': int(timestamp), 'results': self._results}
161         for (name, path) in self._port.repository_paths():
162             contents[name + '-revision'] = self._host.scm().svn_revision(path)
163
164         for key, value in {'branch': branch, 'platform': platform, 'builder-name': builder_name, 'build-number': build_number}.items():
165             if value:
166                 contents[key] = value
167
168         filesystem = self._host.filesystem
169         succeeded = False
170         if source_json_path:
171             try:
172                 source_json_file = filesystem.open_text_file_for_reading(source_json_path)
173                 source_json = json.load(source_json_file)
174                 contents = dict(source_json.items() + contents.items())
175                 succeeded = True
176             except IOError, error:
177                 _log.error("Failed to read %s: %s" % (source_json_path, error))
178             except ValueError, error:
179                 _log.error("Failed to parse %s: %s" % (source_json_path, error))
180             except TypeError, error:
181                 _log.error("Failed to merge JSON files: %s" % error)
182             if not succeeded:
183                 return False
184
185         filesystem.write_text_file(output_json_path, json.dumps(contents))
186         return True
187
188     def _upload_json(self, test_results_server, json_path, file_uploader=FileUploader):
189         uploader = file_uploader("https://%s/api/test/report" % test_results_server, 120)
190         try:
191             response = uploader.upload_single_text_file(self._host.filesystem, 'application/json', json_path)
192         except Exception, error:
193             _log.error("Failed to upload JSON file in 120s: %s" % error)
194             return False
195
196         response_body = [line.strip('\n') for line in response]
197         if response_body != ['OK']:
198             _log.error("Uploaded JSON but got a bad response:")
199             for line in response_body:
200                 _log.error(line)
201             return False
202
203         self._printer.write("JSON file uploaded.")
204         return True
205
206     def _print_status(self, tests, expected, unexpected):
207         if len(tests) == expected + unexpected:
208             status = "Ran %d tests" % len(tests)
209         else:
210             status = "Running %d of %d tests" % (expected + unexpected + 1, len(tests))
211         if unexpected:
212             status += " (%d didn't run)" % unexpected
213         self._printer.write(status)
214
215     def _run_tests_set(self, tests, port):
216         result_count = len(tests)
217         expected = 0
218         unexpected = 0
219         driver = None
220
221         for test in tests:
222             driver = port.create_driver(worker_number=1, no_timeout=True)
223
224             if self._options.pause_before_testing:
225                 driver.start()
226                 if not self._host.user.confirm("Ready to run test?"):
227                     driver.stop()
228                     return unexpected
229
230             relative_test_path = self._host.filesystem.relpath(test, self._base_path)
231             self._printer.write('Running %s (%d of %d)' % (relative_test_path, expected + unexpected + 1, len(tests)))
232
233             is_chromium_style = self._host.filesystem.split(relative_test_path)[0] in self._test_directories_for_chromium_style_tests
234             if self._run_single_test(test, driver, is_chromium_style):
235                 expected = expected + 1
236             else:
237                 unexpected = unexpected + 1
238
239             self._printer.write('')
240
241             driver.stop()
242
243         return unexpected
244
245     _inspector_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
246
247     def _process_chromium_style_test_result(self, test, output):
248         test_failed = False
249         got_a_result = False
250         for line in re.split('\n', output.text):
251             resultLine = self._inspector_result_regex.match(line)
252             if resultLine:
253                 self._results[resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
254                 self._buildbot_output.write("%s\n" % line)
255                 got_a_result = True
256             elif not len(line) == 0:
257                 test_failed = True
258                 self._printer.write("%s" % line)
259         return test_failed or not got_a_result
260
261     _lines_to_ignore_in_parser_result = [
262         re.compile(r'^Running \d+ times$'),
263         re.compile(r'^Ignoring warm-up '),
264         re.compile(r'^Info:'),
265         re.compile(r'^\d+(.\d+)?$'),
266         # Following are for handle existing test like Dromaeo
267         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
268         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
269         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)"""))]
270
271     def _should_ignore_line_in_parser_test_result(self, line):
272         if not line:
273             return True
274         for regex in self._lines_to_ignore_in_parser_result:
275             if regex.search(line):
276                 return True
277         return False
278
279     def _process_parser_test_result(self, test, output):
280         got_a_result = False
281         test_failed = False
282         filesystem = self._host.filesystem
283         category, test_name = filesystem.split(filesystem.relpath(test, self._base_path))
284         test_name = filesystem.splitext(test_name)[0]
285         results = {}
286         keys = ['avg', 'median', 'stdev', 'min', 'max']
287         score_regex = re.compile(r'^(?P<key>' + r'|'.join(keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
288         unit = "ms"
289         for line in re.split('\n', output.text):
290             score = score_regex.match(line)
291             if score:
292                 results[score.group('key')] = float(score.group('value'))
293                 if score.group('unit'):
294                     unit = score.group('unit')
295                 continue
296
297             if not self._should_ignore_line_in_parser_test_result(line):
298                 test_failed = True
299                 self._printer.write("%s" % line)
300
301         if test_failed or set(keys) != set(results.keys()):
302             return True
303         self._results[filesystem.join(category, test_name).replace('\\', '/')] = results
304         self._buildbot_output.write('RESULT %s: %s= %s %s\n' % (category, test_name, results['avg'], unit))
305         self._buildbot_output.write(', '.join(['%s= %s %s' % (key, results[key], unit) for key in keys[1:]]) + '\n')
306         return False
307
308     def _run_single_test(self, test, driver, is_chromium_style):
309         test_failed = False
310         start_time = time.time()
311
312         output = driver.run_test(DriverInput(test, self._options.time_out_ms, None, False))
313
314         if output.text == None:
315             test_failed = True
316         elif output.timeout:
317             self._printer.write('timeout: %s' % test[self._webkit_base_dir_len + 1:])
318             test_failed = True
319         elif output.crash:
320             self._printer.write('crash: %s' % test[self._webkit_base_dir_len + 1:])
321             test_failed = True
322         else:
323             if is_chromium_style:
324                 test_failed = self._process_chromium_style_test_result(test, output)
325             else:
326                 test_failed = self._process_parser_test_result(test, output)
327
328         if len(output.error):
329             self._printer.write('error:\n%s' % output.error)
330             test_failed = True
331
332         if test_failed:
333             self._printer.write('FAILED')
334
335         self._printer.write("Finished: %f s" % (time.time() - start_time))
336
337         return not test_failed