run-perf-tests should record indivisual value instead of statistics
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import sys
40 import time
41
42 # Import for auto-install
43 if sys.platform not in ('cygwin', 'win32'):
44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
46
47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
48 from webkitpy.layout_tests.port.driver import DriverInput
49 from webkitpy.layout_tests.port.driver import DriverOutput
50
51
52 _log = logging.getLogger(__name__)
53
54
55 class PerfTest(object):
56     def __init__(self, port, test_name, path_or_url):
57         self._port = port
58         self._test_name = test_name
59         self._path_or_url = path_or_url
60
61     def test_name(self):
62         return self._test_name
63
64     def path_or_url(self):
65         return self._path_or_url
66
67     def prepare(self, time_out_ms):
68         return True
69
70     def run(self, driver, time_out_ms):
71         output = self.run_single(driver, self.path_or_url(), time_out_ms)
72         if self.run_failed(output):
73             return None
74         return self.parse_output(output)
75
76     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
77         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test), stop_when_done=False)
78
79     def run_failed(self, output):
80         if output.text == None or output.error:
81             pass
82         elif output.timeout:
83             _log.error('timeout: %s' % self.test_name())
84         elif output.crash:
85             _log.error('crash: %s' % self.test_name())
86         else:
87             return False
88
89         if output.error:
90             _log.error('error: %s\n%s' % (self.test_name(), output.error))
91
92         return True
93
94     _lines_to_ignore_in_parser_result = [
95         re.compile(r'^Running \d+ times$'),
96         re.compile(r'^Ignoring warm-up '),
97         re.compile(r'^Info:'),
98         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
99         # Following are for handle existing test like Dromaeo
100         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
101         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
102         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)""")),
103         # Following is for html5.html
104         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/"""))]
105
106     def _should_ignore_line_in_parser_test_result(self, line):
107         if not line:
108             return True
109         for regex in self._lines_to_ignore_in_parser_result:
110             if regex.search(line):
111                 return True
112         return False
113
114     _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
115     _result_classes = ['Time', 'JS Heap', 'Malloc']
116     _result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(_result_classes) + '):')
117     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
118     _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)')
119
120     def parse_output(self, output):
121         test_failed = False
122         results = {}
123         ordered_results_keys = []
124         test_name = re.sub(r'\.\w+$', '', self._test_name)
125         description_string = ""
126         result_class = ""
127         for line in re.split('\n', output.text):
128             description = self._description_regex.match(line)
129             if description:
130                 description_string = description.group('description')
131                 continue
132
133             result_class_match = self._result_class_regex.match(line)
134             if result_class_match:
135                 result_class = result_class_match.group('resultclass')
136                 continue
137
138             score = self._score_regex.match(line)
139             if score:
140                 key = score.group('key')
141                 if ', ' in score.group('value'):
142                     value = [float(number) for number in score.group('value').split(', ')]
143                 else:
144                     value = float(score.group('value'))
145                 unit = score.group('unit')
146                 name = test_name
147                 if result_class != 'Time':
148                     name += ':' + result_class.replace(' ', '')
149                 if name not in ordered_results_keys:
150                     ordered_results_keys.append(name)
151                 results.setdefault(name, {})
152                 results[name]['unit'] = unit
153                 results[name][key] = value
154                 continue
155
156             if not self._should_ignore_line_in_parser_test_result(line):
157                 test_failed = True
158                 _log.error(line)
159
160         if test_failed or set(self._statistics_keys) != set(results[test_name].keys()):
161             return None
162
163         for result_name in ordered_results_keys:
164             if result_name == test_name:
165                 self.output_statistics(result_name, results[result_name], description_string)
166             else:
167                 self.output_statistics(result_name, results[result_name])
168         return results
169
170     def output_statistics(self, test_name, results, description_string=None):
171         unit = results['unit']
172         if description_string:
173             _log.info('DESCRIPTION: %s' % description_string)
174         _log.info('RESULT %s= %s %s' % (test_name.replace(':', ': ').replace('/', ': '), results['avg'], unit))
175         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
176
177
178 class ChromiumStylePerfTest(PerfTest):
179     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
180
181     def __init__(self, port, test_name, path_or_url):
182         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
183
184     def parse_output(self, output):
185         test_failed = False
186         results = {}
187         for line in re.split('\n', output.text):
188             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
189             if resultLine:
190                 # FIXME: Store the unit
191                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
192                 _log.info(line)
193             elif not len(line) == 0:
194                 test_failed = True
195                 _log.error(line)
196         return results if results and not test_failed else None
197
198
199 class PageLoadingPerfTest(PerfTest):
200     def __init__(self, port, test_name, path_or_url):
201         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
202
203     def run(self, driver, time_out_ms):
204         test_times = []
205
206         for i in range(0, 20):
207             output = self.run_single(driver, self.path_or_url(), time_out_ms)
208             if not output or self.run_failed(output):
209                 return None
210             if i == 0:
211                 continue
212             test_times.append(output.test_time * 1000)
213
214         sorted_test_times = sorted(test_times)
215
216         # Compute the mean and variance using a numerically stable algorithm.
217         squareSum = 0
218         mean = 0
219         valueSum = sum(sorted_test_times)
220         for i, time in enumerate(sorted_test_times):
221             delta = time - mean
222             sweep = i + 1.0
223             mean += delta / sweep
224             squareSum += delta * delta * (i / sweep)
225
226         middle = int(len(test_times) / 2)
227         results = {'values': test_times,
228             'avg': mean,
229             'min': sorted_test_times[0],
230             'max': sorted_test_times[-1],
231             'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2,
232             'stdev': math.sqrt(squareSum),
233             'unit': 'ms'}
234         self.output_statistics(self.test_name(), results, '')
235         return {self.test_name(): results}
236
237
238 class ReplayServer(object):
239     def __init__(self, archive, record):
240         self._process = None
241
242         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
243
244         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
245         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
246         if record:
247             args.append('--record')
248         args.append(archive)
249
250         self._process = subprocess.Popen(args)
251
252     def wait_until_ready(self):
253         for i in range(0, 3):
254             try:
255                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
256                 connection.close()
257                 return True
258             except socket.error:
259                 time.sleep(1)
260                 continue
261         return False
262
263     def stop(self):
264         if self._process:
265             self._process.send_signal(signal.SIGINT)
266             self._process.wait()
267         self._process = None
268
269     def __del__(self):
270         self.stop()
271
272
273 class ReplayPerfTest(PageLoadingPerfTest):
274     def __init__(self, port, test_name, path_or_url):
275         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
276
277     def _start_replay_server(self, archive, record):
278         try:
279             return ReplayServer(archive, record)
280         except OSError as error:
281             if error.errno == errno.ENOENT:
282                 _log.error("Replay tests require web-page-replay.")
283             else:
284                 raise error
285
286     def prepare(self, time_out_ms):
287         filesystem = self._port.host.filesystem
288         path_without_ext = filesystem.splitext(self.path_or_url())[0]
289
290         self._archive_path = filesystem.join(path_without_ext + '.wpr')
291         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
292         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
293
294         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
295             _log.info("Replay ready for %s" % self._archive_path)
296             return True
297
298         _log.info("Preparing replay for %s" % self.test_name())
299
300         driver = self._port.create_driver(worker_number=1, no_timeout=True)
301         try:
302             output = self.run_single(driver, self._archive_path, time_out_ms, record=True)
303         finally:
304             driver.stop()
305
306         if not output or not filesystem.isfile(self._archive_path):
307             _log.error("Failed to prepare a replay for %s" % self.test_name())
308             return False
309
310         _log.info("Prepared replay for %s" % self.test_name())
311
312         return True
313
314     def run_single(self, driver, url, time_out_ms, record=False):
315         server = self._start_replay_server(self._archive_path, record)
316         if not server:
317             _log.error("Web page replay didn't start.")
318             return None
319
320         try:
321             _log.debug("Waiting for Web page replay to start.")
322             if not server.wait_until_ready():
323                 _log.error("Web page replay didn't start.")
324                 return None
325
326             _log.debug("Web page replay started. Loading the page.")
327             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
328             if self.run_failed(output):
329                 return None
330
331             if not output.image:
332                 _log.error("Loading the page did not generate image results")
333                 _log.error(output.text)
334                 return None
335
336             filesystem = self._port.host.filesystem
337             dirname = filesystem.dirname(self._archive_path)
338             filename = filesystem.split(self._archive_path)[1]
339             writer = TestResultWriter(filesystem, self._port, dirname, filename)
340             if record:
341                 writer.write_image_files(actual_image=None, expected_image=output.image)
342             else:
343                 writer.write_image_files(actual_image=output.image, expected_image=None)
344
345             return output
346         finally:
347             server.stop()
348
349
350 class PerfTestFactory(object):
351
352     _pattern_map = [
353         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
354         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
355         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
356     ]
357
358     @classmethod
359     def create_perf_test(cls, port, test_name, path):
360         for (pattern, test_class) in cls._pattern_map:
361             if pattern.match(test_name):
362                 return test_class(port, test_name, path)
363         return PerfTest(port, test_name, path)