Add public page loading performance tests using web-page-replay
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import time
40
41 # Import for auto-install
42 import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
43
44 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
45 from webkitpy.layout_tests.port.driver import DriverInput
46 from webkitpy.layout_tests.port.driver import DriverOutput
47
48
49 _log = logging.getLogger(__name__)
50
51
52 class PerfTest(object):
53     def __init__(self, port, test_name, path_or_url):
54         self._port = port
55         self._test_name = test_name
56         self._path_or_url = path_or_url
57
58     def test_name(self):
59         return self._test_name
60
61     def path_or_url(self):
62         return self._path_or_url
63
64     def prepare(self, time_out_ms):
65         return True
66
67     def run(self, driver, time_out_ms):
68         output = self.run_single(driver, self.path_or_url(), time_out_ms)
69         if self.run_failed(output):
70             return None
71         return self.parse_output(output)
72
73     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
74         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test))
75
76     def run_failed(self, output):
77         if output.text == None or output.error:
78             pass
79         elif output.timeout:
80             _log.error('timeout: %s' % self.test_name())
81         elif output.crash:
82             _log.error('crash: %s' % self.test_name())
83         else:
84             return False
85
86         if output.error:
87             _log.error('error: %s\n%s' % (self.test_name(), output.error))
88
89         return True
90
91     _lines_to_ignore_in_parser_result = [
92         re.compile(r'^Running \d+ times$'),
93         re.compile(r'^Ignoring warm-up '),
94         re.compile(r'^Info:'),
95         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms))?$'),
96         # Following are for handle existing test like Dromaeo
97         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
98         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
99         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)"""))]
100
101     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max']
102
103     def _should_ignore_line_in_parser_test_result(self, line):
104         if not line:
105             return True
106         for regex in self._lines_to_ignore_in_parser_result:
107             if regex.search(line):
108                 return True
109         return False
110
111     def parse_output(self, output):
112         got_a_result = False
113         test_failed = False
114         results = {}
115         score_regex = re.compile(r'^(?P<key>' + r'|'.join(self._statistics_keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
116         description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
117         description_string = ""
118         unit = "ms"
119
120         for line in re.split('\n', output.text):
121             description = description_regex.match(line)
122             if description:
123                 description_string = description.group('description')
124                 continue
125
126             score = score_regex.match(line)
127             if score:
128                 results[score.group('key')] = float(score.group('value'))
129                 if score.group('unit'):
130                     unit = score.group('unit')
131                 continue
132
133             if not self._should_ignore_line_in_parser_test_result(line):
134                 test_failed = True
135                 _log.error(line)
136
137         if test_failed or set(self._statistics_keys) != set(results.keys()):
138             return None
139
140         results['unit'] = unit
141
142         test_name = re.sub(r'\.\w+$', '', self._test_name)
143         self.output_statistics(test_name, results, description_string)
144
145         return {test_name: results}
146
147     def output_statistics(self, test_name, results, description_string):
148         unit = results['unit']
149         if description_string:
150             _log.info('DESCRIPTION: %s' % description_string)
151         _log.info('RESULT %s= %s %s' % (test_name.replace('/', ': '), results['avg'], unit))
152         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:]]))
153
154
155 class ChromiumStylePerfTest(PerfTest):
156     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
157
158     def __init__(self, port, test_name, path_or_url):
159         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
160
161     def parse_output(self, output):
162         test_failed = False
163         got_a_result = False
164         results = {}
165         for line in re.split('\n', output.text):
166             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
167             if resultLine:
168                 # FIXME: Store the unit
169                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
170                 _log.info(line)
171             elif not len(line) == 0:
172                 test_failed = True
173                 _log.error(line)
174         return results if results and not test_failed else None
175
176
177 class PageLoadingPerfTest(PerfTest):
178     def __init__(self, port, test_name, path_or_url):
179         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
180
181     def run(self, driver, time_out_ms):
182         test_times = []
183
184         for i in range(0, 20):
185             output = self.run_single(driver, self.path_or_url(), time_out_ms)
186             if not output or self.run_failed(output):
187                 return None
188             if i == 0:
189                 continue
190             test_times.append(output.test_time * 1000)
191
192         test_times = sorted(test_times)
193
194         # Compute the mean and variance using a numerically stable algorithm.
195         squareSum = 0
196         mean = 0
197         valueSum = sum(test_times)
198         for i, time in enumerate(test_times):
199             delta = time - mean
200             sweep = i + 1.0
201             mean += delta / sweep
202             squareSum += delta * delta * (i / sweep)
203
204         middle = int(len(test_times) / 2)
205         results = {'avg': mean,
206             'min': min(test_times),
207             'max': max(test_times),
208             'median': test_times[middle] if len(test_times) % 2 else (test_times[middle - 1] + test_times[middle]) / 2,
209             'stdev': math.sqrt(squareSum),
210             'unit': 'ms'}
211         self.output_statistics(self.test_name(), results, '')
212         return {self.test_name(): results}
213
214
215 class ReplayServer(object):
216     def __init__(self, archive, record):
217         self._process = None
218
219         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8413
220
221         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
222         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8413', '--use_closest_match', '--log_level', 'warning']
223         if record:
224             args.append('--record')
225         args.append(archive)
226
227         self._process = subprocess.Popen(args)
228
229     def wait_until_ready(self):
230         for i in range(0, 10):
231             try:
232                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
233                 connection.close()
234                 return True
235             except socket.error:
236                 time.sleep(1)
237                 continue
238         return False
239
240     def stop(self):
241         if self._process:
242             self._process.send_signal(signal.SIGINT)
243             self._process.wait()
244         self._process = None
245
246     def __del__(self):
247         self.stop()
248
249
250 class ReplayPerfTest(PageLoadingPerfTest):
251     def __init__(self, port, test_name, path_or_url):
252         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
253
254     def _start_replay_server(self, archive, record):
255         try:
256             return ReplayServer(archive, record)
257         except OSError as error:
258             if error.errno == errno.ENOENT:
259                 _log.error("Replay tests require web-page-replay.")
260             else:
261                 raise error
262
263     def prepare(self, time_out_ms):
264         filesystem = self._port.host.filesystem
265         path_without_ext = filesystem.splitext(self.path_or_url())[0]
266
267         self._archive_path = filesystem.join(path_without_ext + '.wpr')
268         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
269         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
270
271         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
272             _log.info("Replay ready for %s" % self._archive_path)
273             return True
274
275         _log.info("Preparing replay for %s" % self.test_name())
276
277         driver = self._port.create_driver(worker_number=1, no_timeout=True)
278         try:
279             output = self.run_single(driver, self._url, time_out_ms, record=True)
280         finally:
281             driver.stop()
282
283         if not output or not filesystem.isfile(self._archive_path):
284             _log.error("Failed to prepare a replay for %s" % self.test_name())
285             return False
286
287         _log.info("Prepared replay for %s" % self.test_name())
288
289         return True
290
291     def run_single(self, driver, url, time_out_ms, record=False):
292         server = self._start_replay_server(self._archive_path, record)
293         if not server:
294             _log.error("Web page replay didn't start.")
295             return None
296
297         try:
298             if not server.wait_until_ready():
299                 _log.error("Web page replay didn't start.")
300                 return None
301
302             super(ReplayPerfTest, self).run_single(driver, "about:blank", time_out_ms)
303             _log.debug("Loading the page")
304
305             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
306             if self.run_failed(output):
307                 return None
308
309             if not output.image:
310                 _log.error("Loading the page did not generate image results")
311                 _log.error(output.text)
312                 return None
313
314             filesystem = self._port.host.filesystem
315             dirname = filesystem.dirname(url)
316             filename = filesystem.split(url)[1]
317             writer = TestResultWriter(filesystem, self._port, dirname, filename)
318             if record:
319                 writer.write_image_files(actual_image=None, expected_image=output.image)
320             else:
321                 writer.write_image_files(actual_image=output.image, expected_image=None)
322
323             return output
324         finally:
325             server.stop()
326
327
328 class PerfTestFactory(object):
329
330     _pattern_map = [
331         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
332         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
333         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
334     ]
335
336     @classmethod
337     def create_perf_test(cls, port, test_name, path):
338         for (pattern, test_class) in cls._pattern_map:
339             if pattern.match(test_name):
340                 return test_class(port, test_name, path)
341         return PerfTest(port, test_name, path)