Force GC between PageLoad tests.
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import sys
40 import time
41
42 # Import for auto-install
43 if sys.platform not in ('cygwin', 'win32'):
44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
46
47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
48 from webkitpy.layout_tests.port.driver import DriverInput
49 from webkitpy.layout_tests.port.driver import DriverOutput
50
51
52 _log = logging.getLogger(__name__)
53
54
55 class PerfTest(object):
56     def __init__(self, port, test_name, path_or_url):
57         self._port = port
58         self._test_name = test_name
59         self._path_or_url = path_or_url
60
61     def test_name(self):
62         return self._test_name
63
64     def path_or_url(self):
65         return self._path_or_url
66
67     def prepare(self, time_out_ms):
68         return True
69
70     def run(self, driver, time_out_ms):
71         output = self.run_single(driver, self.path_or_url(), time_out_ms)
72         if self.run_failed(output):
73             return None
74         return self.parse_output(output)
75
76     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
77         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test), stop_when_done=False)
78
79     def run_failed(self, output):
80         if output.text == None or output.error:
81             pass
82         elif output.timeout:
83             _log.error('timeout: %s' % self.test_name())
84         elif output.crash:
85             _log.error('crash: %s' % self.test_name())
86         else:
87             return False
88
89         if output.error:
90             _log.error('error: %s\n%s' % (self.test_name(), output.error))
91
92         return True
93
94     _lines_to_ignore_in_parser_result = [
95         re.compile(r'^Running \d+ times$'),
96         re.compile(r'^Ignoring warm-up '),
97         re.compile(r'^Info:'),
98         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
99         # Following are for handle existing test like Dromaeo
100         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
101         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
102         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)""")),
103         # Following is for html5.html
104         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/"""))]
105
106     def _should_ignore_line_in_parser_test_result(self, line):
107         if not line:
108             return True
109         for regex in self._lines_to_ignore_in_parser_result:
110             if regex.search(line):
111                 return True
112         return False
113
114     _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
115     _result_classes = ['Time', 'JS Heap', 'Malloc']
116     _result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(_result_classes) + '):')
117     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
118     _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)')
119
120     def parse_output(self, output):
121         test_failed = False
122         results = {}
123         ordered_results_keys = []
124         test_name = re.sub(r'\.\w+$', '', self._test_name)
125         description_string = ""
126         result_class = ""
127         for line in re.split('\n', output.text):
128             description = self._description_regex.match(line)
129             if description:
130                 description_string = description.group('description')
131                 continue
132
133             result_class_match = self._result_class_regex.match(line)
134             if result_class_match:
135                 result_class = result_class_match.group('resultclass')
136                 continue
137
138             score = self._score_regex.match(line)
139             if score:
140                 key = score.group('key')
141                 if key == 'values':
142                     value = [float(number) for number in score.group('value').split(', ')]
143                 else:
144                     value = float(score.group('value'))
145                 unit = score.group('unit')
146                 name = test_name
147                 if result_class != 'Time':
148                     name += ':' + result_class.replace(' ', '')
149                 if name not in ordered_results_keys:
150                     ordered_results_keys.append(name)
151                 results.setdefault(name, {})
152                 results[name]['unit'] = unit
153                 results[name][key] = value
154                 continue
155
156             if not self._should_ignore_line_in_parser_test_result(line):
157                 test_failed = True
158                 _log.error(line)
159
160         if test_failed:
161             return None
162
163         if set(self._statistics_keys) != set(results[test_name].keys() + ['values']):
164             # values is not provided by Dromaeo tests.
165             _log.error("The test didn't report all statistics.")
166             return None
167
168         for result_name in ordered_results_keys:
169             if result_name == test_name:
170                 self.output_statistics(result_name, results[result_name], description_string)
171             else:
172                 self.output_statistics(result_name, results[result_name])
173         return results
174
175     def output_statistics(self, test_name, results, description_string=None):
176         unit = results['unit']
177         if description_string:
178             _log.info('DESCRIPTION: %s' % description_string)
179         _log.info('RESULT %s= %s %s' % (test_name.replace(':', ': ').replace('/', ': '), results['avg'], unit))
180         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
181
182
183 class ChromiumStylePerfTest(PerfTest):
184     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
185
186     def __init__(self, port, test_name, path_or_url):
187         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
188
189     def parse_output(self, output):
190         test_failed = False
191         results = {}
192         for line in re.split('\n', output.text):
193             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
194             if resultLine:
195                 # FIXME: Store the unit
196                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
197                 _log.info(line)
198             elif not len(line) == 0:
199                 test_failed = True
200                 _log.error(line)
201         return results if results and not test_failed else None
202
203
204 class PageLoadingPerfTest(PerfTest):
205     _FORCE_GC_FILE = 'resources/force-gc.html'
206
207     def __init__(self, port, test_name, path_or_url):
208         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
209         self.force_gc_test = self._port.host.filesystem.join(self._port.perf_tests_dir(), self._FORCE_GC_FILE)
210
211     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
212         # Force GC to prevent pageload noise. See https://bugs.webkit.org/show_bug.cgi?id=98203
213         super(PageLoadingPerfTest, self).run_single(driver, self.force_gc_test, time_out_ms, False)
214         return super(PageLoadingPerfTest, self).run_single(driver, path_or_url, time_out_ms, should_run_pixel_test)
215
216     def run(self, driver, time_out_ms):
217         test_times = []
218
219         for i in range(0, 20):
220             output = self.run_single(driver, self.path_or_url(), time_out_ms)
221             if not output or self.run_failed(output):
222                 return None
223             if i == 0:
224                 continue
225             test_times.append(output.test_time * 1000)
226
227         sorted_test_times = sorted(test_times)
228
229         # Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
230         squareSum = 0
231         mean = 0
232         for i, time in enumerate(sorted_test_times):
233             delta = time - mean
234             sweep = i + 1.0
235             mean += delta / sweep
236             squareSum += delta * (time - mean)
237
238         middle = int(len(test_times) / 2)
239         results = {'values': test_times,
240             'avg': mean,
241             'min': sorted_test_times[0],
242             'max': sorted_test_times[-1],
243             'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2,
244             'stdev': math.sqrt(squareSum / (len(sorted_test_times) - 1)),
245             'unit': 'ms'}
246         self.output_statistics(self.test_name(), results, '')
247         return {self.test_name(): results}
248
249
250 class ReplayServer(object):
251     def __init__(self, archive, record):
252         self._process = None
253
254         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
255
256         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
257         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
258         if record:
259             args.append('--record')
260         args.append(archive)
261
262         self._process = subprocess.Popen(args)
263
264     def wait_until_ready(self):
265         for i in range(0, 3):
266             try:
267                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
268                 connection.close()
269                 return True
270             except socket.error:
271                 time.sleep(1)
272                 continue
273         return False
274
275     def stop(self):
276         if self._process:
277             self._process.send_signal(signal.SIGINT)
278             self._process.wait()
279         self._process = None
280
281     def __del__(self):
282         self.stop()
283
284
285 class ReplayPerfTest(PageLoadingPerfTest):
286     def __init__(self, port, test_name, path_or_url):
287         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
288
289     def _start_replay_server(self, archive, record):
290         try:
291             return ReplayServer(archive, record)
292         except OSError as error:
293             if error.errno == errno.ENOENT:
294                 _log.error("Replay tests require web-page-replay.")
295             else:
296                 raise error
297
298     def prepare(self, time_out_ms):
299         filesystem = self._port.host.filesystem
300         path_without_ext = filesystem.splitext(self.path_or_url())[0]
301
302         self._archive_path = filesystem.join(path_without_ext + '.wpr')
303         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
304         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
305
306         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
307             _log.info("Replay ready for %s" % self._archive_path)
308             return True
309
310         _log.info("Preparing replay for %s" % self.test_name())
311
312         driver = self._port.create_driver(worker_number=1, no_timeout=True)
313         try:
314             output = self.run_single(driver, self._archive_path, time_out_ms, record=True)
315         finally:
316             driver.stop()
317
318         if not output or not filesystem.isfile(self._archive_path):
319             _log.error("Failed to prepare a replay for %s" % self.test_name())
320             return False
321
322         _log.info("Prepared replay for %s" % self.test_name())
323
324         return True
325
326     def run_single(self, driver, url, time_out_ms, record=False):
327         server = self._start_replay_server(self._archive_path, record)
328         if not server:
329             _log.error("Web page replay didn't start.")
330             return None
331
332         try:
333             _log.debug("Waiting for Web page replay to start.")
334             if not server.wait_until_ready():
335                 _log.error("Web page replay didn't start.")
336                 return None
337
338             _log.debug("Web page replay started. Loading the page.")
339             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
340             if self.run_failed(output):
341                 return None
342
343             if not output.image:
344                 _log.error("Loading the page did not generate image results")
345                 _log.error(output.text)
346                 return None
347
348             filesystem = self._port.host.filesystem
349             dirname = filesystem.dirname(self._archive_path)
350             filename = filesystem.split(self._archive_path)[1]
351             writer = TestResultWriter(filesystem, self._port, dirname, filename)
352             if record:
353                 writer.write_image_files(actual_image=None, expected_image=output.image)
354             else:
355                 writer.write_image_files(actual_image=output.image, expected_image=None)
356
357             return output
358         finally:
359             server.stop()
360
361
362 class PerfTestFactory(object):
363
364     _pattern_map = [
365         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
366         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
367         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
368     ]
369
370     @classmethod
371     def create_perf_test(cls, port, test_name, path):
372         for (pattern, test_class) in cls._pattern_map:
373             if pattern.match(test_name):
374                 return test_class(port, test_name, path)
375         return PerfTest(port, test_name, path)