254baa74c83ef784053de45ae3634df5f355bff0
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 # Copyright (C) 2012 Zoltan Horvath, Adobe Systems Incorporated. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import sys
40 import time
41
42 # Import for auto-install
43 if sys.platform not in ('cygwin', 'win32'):
44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
46
47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
48 from webkitpy.layout_tests.port.driver import DriverInput
49 from webkitpy.layout_tests.port.driver import DriverOutput
50
51
52 _log = logging.getLogger(__name__)
53
54
55 class PerfTest(object):
56     def __init__(self, port, test_name, test_path):
57         self._port = port
58         self._test_name = test_name
59         self._test_path = test_path
60         self._description = None
61
62     def test_name(self):
63         return self._test_name
64
65     def test_path(self):
66         return self._test_path
67
68     def description(self):
69         return self._description
70
71     def prepare(self, time_out_ms):
72         return True
73
74     def run(self, time_out_ms):
75         driver = self._port.create_driver(worker_number=0, no_timeout=True)
76         try:
77             return self._run_with_driver(driver, time_out_ms)
78         finally:
79             driver.stop()
80
81     def _run_with_driver(self, driver, time_out_ms):
82         output = self.run_single(driver, self.test_path(), time_out_ms)
83         self._filter_output(output)
84         if self.run_failed(output):
85             return None
86
87         results = self.parse_output(output)
88         if not results:
89             return None
90
91         if not self._port.get_option('profile'):
92             if self._description:
93                 _log.info('DESCRIPTION: %s' % self._description)
94             for result_name in sorted(results.keys()):
95                 self.output_statistics(result_name, results[result_name])
96
97         return results
98
99     def run_single(self, driver, test_path, time_out_ms, should_run_pixel_test=False):
100         return driver.run_test(DriverInput(test_path, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test), stop_when_done=False)
101
102     def run_failed(self, output):
103         if output.text == None or output.error:
104             pass
105         elif output.timeout:
106             _log.error('timeout: %s' % self.test_name())
107         elif output.crash:
108             _log.error('crash: %s' % self.test_name())
109         else:
110             return False
111
112         if output.error:
113             _log.error('error: %s\n%s' % (self.test_name(), output.error))
114
115         return True
116
117     def _should_ignore_line(self, regexps, line):
118         if not line:
119             return True
120         for regexp in regexps:
121             if regexp.search(line):
122                 return True
123         return False
124
125     _lines_to_ignore_in_stderr = [
126         re.compile(r'^Unknown option:'),
127         re.compile(r'^\[WARNING:proxy_service.cc'),
128         re.compile(r'^\[INFO:'),
129     ]
130
131     def _should_ignore_line_in_stderr(self, line):
132         return self._should_ignore_line(self._lines_to_ignore_in_stderr, line)
133
134     _lines_to_ignore_in_parser_result = [
135         re.compile(r'^Running \d+ times$'),
136         re.compile(r'^Ignoring warm-up '),
137         re.compile(r'^Info:'),
138         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
139         # Following are for handle existing test like Dromaeo
140         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
141         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
142         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)""")),
143         # Following is for html5.html
144         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/""")),
145         # Following is for Parser/html-parser.html
146         re.compile(re.escape("""CONSOLE MESSAGE: Blocked script execution in 'html-parser.html' because the document's frame is sandboxed and the 'allow-scripts' permission is not set.""")),
147         # Dromaeo reports values for subtests. Ignore them for now.
148         re.compile(r'(?P<name>.+): \[(?P<values>(\d+(.\d+)?,\s+)*\d+(.\d+)?)\]'),
149     ]
150
151     def _should_ignore_line_in_parser_test_result(self, line):
152         return self._should_ignore_line(self._lines_to_ignore_in_parser_result, line)
153
154     def _filter_output(self, output):
155         if output.error:
156             filtered_error = '\n'.join([line for line in re.split('\n', output.error) if not self._should_ignore_line_in_stderr(line)])
157             output.error = filtered_error if filtered_error else None
158         if output.text:
159             output.text = '\n'.join([line for line in re.split('\n', output.text) if not self._should_ignore_line_in_parser_test_result(line)])
160
161     _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
162     _result_classes = ['Time', 'JS Heap', 'Malloc']
163     _result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(_result_classes) + '):')
164     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
165     _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)')
166
167     def parse_output(self, output):
168         test_failed = False
169         results = {}
170         test_name = re.sub(r'\.\w+$', '', self._test_name)
171         result_class = ""
172         for line in re.split('\n', output.text):
173             if not line:
174                 continue
175
176             description_match = self._description_regex.match(line)
177             if description_match:
178                 self._description = description_match.group('description')
179                 continue
180
181             result_class_match = self._result_class_regex.match(line)
182             if result_class_match:
183                 result_class = result_class_match.group('resultclass')
184                 continue
185
186             score = self._score_regex.match(line)
187             if score:
188                 key = score.group('key')
189                 if key == 'values':
190                     value = [float(number) for number in score.group('value').split(', ')]
191                 else:
192                     value = float(score.group('value'))
193                 unit = score.group('unit')
194                 name = test_name
195                 if result_class != 'Time':
196                     name += ':' + result_class.replace(' ', '')
197                 results.setdefault(name, {})
198                 results[name]['unit'] = unit
199                 results[name][key] = value
200                 continue
201
202             test_failed = True
203             _log.error('ERROR: ' + line)
204
205         if test_failed:
206             return None
207
208         if set(self._statistics_keys) != set(results[test_name].keys()):
209             _log.error("The test didn't report all statistics.")
210             return None
211
212         return results
213
214     @staticmethod
215     def compute_statistics(values):
216         sorted_values = sorted(values)
217
218         # Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
219         squareSum = 0
220         mean = 0
221         for i, time in enumerate(sorted_values):
222             delta = time - mean
223             sweep = i + 1.0
224             mean += delta / sweep
225             squareSum += delta * (time - mean)
226
227         middle = int(len(sorted_values) / 2)
228         result = {'avg': sum(sorted_values) / len(values),
229             'min': sorted_values[0],
230             'max': sorted_values[-1],
231             'median': sorted_values[middle] if len(sorted_values) % 2 else (sorted_values[middle - 1] + sorted_values[middle]) / 2,
232             'stdev': math.sqrt(squareSum / (len(sorted_values) - 1))}
233         return result
234
235     def output_statistics(self, test_name, results):
236         unit = results['unit']
237         _log.info('RESULT %s= %s %s' % (test_name.replace(':', ': ').replace('/', ': '), results['avg'], unit))
238         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
239
240
241 class ChromiumStylePerfTest(PerfTest):
242     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
243
244     def __init__(self, port, test_name, test_path):
245         super(ChromiumStylePerfTest, self).__init__(port, test_name, test_path)
246
247     def _run_with_driver(self, driver, time_out_ms):
248         output = self.run_single(driver, self.test_path(), time_out_ms)
249         self._filter_output(output)
250         if self.run_failed(output):
251             return None
252
253         return self.parse_and_log_output(output)
254
255     def parse_and_log_output(self, output):
256         test_failed = False
257         results = {}
258         for line in re.split('\n', output.text):
259             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
260             if resultLine:
261                 # FIXME: Store the unit
262                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
263                 _log.info(line)
264             elif not len(line) == 0:
265                 test_failed = True
266                 _log.error(line)
267         return results if results and not test_failed else None
268
269
270 class PageLoadingPerfTest(PerfTest):
271     _FORCE_GC_FILE = 'resources/force-gc.html'
272
273     def __init__(self, port, test_name, test_path):
274         super(PageLoadingPerfTest, self).__init__(port, test_name, test_path)
275         self.force_gc_test = self._port.host.filesystem.join(self._port.perf_tests_dir(), self._FORCE_GC_FILE)
276
277     def run_single(self, driver, test_path, time_out_ms, should_run_pixel_test=False):
278         # Force GC to prevent pageload noise. See https://bugs.webkit.org/show_bug.cgi?id=98203
279         super(PageLoadingPerfTest, self).run_single(driver, self.force_gc_test, time_out_ms, False)
280         return super(PageLoadingPerfTest, self).run_single(driver, test_path, time_out_ms, should_run_pixel_test)
281
282     def _run_with_driver(self, driver, time_out_ms):
283         results = {}
284         results.setdefault(self.test_name(), {'unit': 'ms', 'values': []})
285
286         for i in range(0, 20):
287             output = self.run_single(driver, self.test_path(), time_out_ms)
288             if not output or self.run_failed(output):
289                 return None
290             if i == 0:
291                 continue
292
293             results[self.test_name()]['values'].append(output.test_time * 1000)
294
295             if not output.measurements:
296                 continue
297
298             for result_class, result in output.measurements.items():
299                 name = self.test_name() + ':' + result_class
300                 if not name in results:
301                     results.setdefault(name, {'values': []})
302                 results[name]['values'].append(result)
303                 if result_class == 'Malloc' or result_class == 'JSHeap':
304                     results[name]['unit'] = 'bytes'
305
306         for result_class in results.keys():
307             results[result_class].update(self.compute_statistics(results[result_class]['values']))
308             self.output_statistics(result_class, results[result_class])
309
310         return results
311
312
313 class ReplayServer(object):
314     def __init__(self, archive, record):
315         self._process = None
316
317         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
318
319         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
320         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
321         if record:
322             args.append('--record')
323         args.append(archive)
324
325         self._process = subprocess.Popen(args)
326
327     def wait_until_ready(self):
328         for i in range(0, 3):
329             try:
330                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
331                 connection.close()
332                 return True
333             except socket.error:
334                 time.sleep(1)
335                 continue
336         return False
337
338     def stop(self):
339         if self._process:
340             self._process.send_signal(signal.SIGINT)
341             self._process.wait()
342         self._process = None
343
344     def __del__(self):
345         self.stop()
346
347
348 class ReplayPerfTest(PageLoadingPerfTest):
349     def __init__(self, port, test_name, test_path):
350         super(ReplayPerfTest, self).__init__(port, test_name, test_path)
351
352     def _start_replay_server(self, archive, record):
353         try:
354             return ReplayServer(archive, record)
355         except OSError as error:
356             if error.errno == errno.ENOENT:
357                 _log.error("Replay tests require web-page-replay.")
358             else:
359                 raise error
360
361     def prepare(self, time_out_ms):
362         filesystem = self._port.host.filesystem
363         path_without_ext = filesystem.splitext(self.test_path())[0]
364
365         self._archive_path = filesystem.join(path_without_ext + '.wpr')
366         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
367         self._url = filesystem.read_text_file(self.test_path()).split('\n')[0]
368
369         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
370             _log.info("Replay ready for %s" % self._archive_path)
371             return True
372
373         _log.info("Preparing replay for %s" % self.test_name())
374
375         driver = self._port.create_driver(worker_number=0, no_timeout=True)
376         try:
377             output = self.run_single(driver, self._archive_path, time_out_ms, record=True)
378         finally:
379             driver.stop()
380
381         if not output or not filesystem.isfile(self._archive_path):
382             _log.error("Failed to prepare a replay for %s" % self.test_name())
383             return False
384
385         _log.info("Prepared replay for %s" % self.test_name())
386
387         return True
388
389     def run_single(self, driver, url, time_out_ms, record=False):
390         server = self._start_replay_server(self._archive_path, record)
391         if not server:
392             _log.error("Web page replay didn't start.")
393             return None
394
395         try:
396             _log.debug("Waiting for Web page replay to start.")
397             if not server.wait_until_ready():
398                 _log.error("Web page replay didn't start.")
399                 return None
400
401             _log.debug("Web page replay started. Loading the page.")
402             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
403             if self.run_failed(output):
404                 return None
405
406             if not output.image:
407                 _log.error("Loading the page did not generate image results")
408                 _log.error(output.text)
409                 return None
410
411             filesystem = self._port.host.filesystem
412             dirname = filesystem.dirname(self._archive_path)
413             filename = filesystem.split(self._archive_path)[1]
414             writer = TestResultWriter(filesystem, self._port, dirname, filename)
415             if record:
416                 writer.write_image_files(actual_image=None, expected_image=output.image)
417             else:
418                 writer.write_image_files(actual_image=output.image, expected_image=None)
419
420             return output
421         finally:
422             server.stop()
423
424
425 class PerfTestFactory(object):
426
427     _pattern_map = [
428         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
429         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
430     ]
431
432     @classmethod
433     def create_perf_test(cls, port, test_name, path):
434         for (pattern, test_class) in cls._pattern_map:
435             if pattern.match(test_name):
436                 return test_class(port, test_name, path)
437         return PerfTest(port, test_name, path)