Unreviewed: Back out accidentally checked in debug print which broke a test
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import sys
40 import time
41
42 # Import for auto-install
43 if sys.platform != 'win32':
44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
46
47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
48 from webkitpy.layout_tests.port.driver import DriverInput
49 from webkitpy.layout_tests.port.driver import DriverOutput
50
51
52 _log = logging.getLogger(__name__)
53
54
55 class PerfTest(object):
56     def __init__(self, port, test_name, path_or_url):
57         self._port = port
58         self._test_name = test_name
59         self._path_or_url = path_or_url
60
61     def test_name(self):
62         return self._test_name
63
64     def path_or_url(self):
65         return self._path_or_url
66
67     def prepare(self, time_out_ms):
68         return True
69
70     def run(self, driver, time_out_ms):
71         output = self.run_single(driver, self.path_or_url(), time_out_ms)
72         if self.run_failed(output):
73             return None
74         return self.parse_output(output)
75
76     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
77         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test))
78
79     def run_failed(self, output):
80         if output.text == None or output.error:
81             pass
82         elif output.timeout:
83             _log.error('timeout: %s' % self.test_name())
84         elif output.crash:
85             _log.error('crash: %s' % self.test_name())
86         else:
87             return False
88
89         if output.error:
90             _log.error('error: %s\n%s' % (self.test_name(), output.error))
91
92         return True
93
94     _lines_to_ignore_in_parser_result = [
95         re.compile(r'^Running \d+ times$'),
96         re.compile(r'^Ignoring warm-up '),
97         re.compile(r'^Info:'),
98         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
99         # Following are for handle existing test like Dromaeo
100         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
101         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
102         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)"""))]
103
104     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max']
105
106     def _should_ignore_line_in_parser_test_result(self, line):
107         if not line:
108             return True
109         for regex in self._lines_to_ignore_in_parser_result:
110             if regex.search(line):
111                 return True
112         return False
113
114     def parse_output(self, output):
115         got_a_result = False
116         test_failed = False
117         results = {}
118         score_regex = re.compile(r'^(?P<key>' + r'|'.join(self._statistics_keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
119         description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
120         description_string = ""
121         unit = "ms"
122
123         for line in re.split('\n', output.text):
124             description = description_regex.match(line)
125             if description:
126                 description_string = description.group('description')
127                 continue
128
129             score = score_regex.match(line)
130             if score:
131                 results[score.group('key')] = float(score.group('value'))
132                 if score.group('unit'):
133                     unit = score.group('unit')
134                 continue
135
136             if not self._should_ignore_line_in_parser_test_result(line):
137                 test_failed = True
138                 _log.error(line)
139
140         if test_failed or set(self._statistics_keys) != set(results.keys()):
141             return None
142
143         results['unit'] = unit
144
145         test_name = re.sub(r'\.\w+$', '', self._test_name)
146         self.output_statistics(test_name, results, description_string)
147
148         return {test_name: results}
149
150     def output_statistics(self, test_name, results, description_string):
151         unit = results['unit']
152         if description_string:
153             _log.info('DESCRIPTION: %s' % description_string)
154         _log.info('RESULT %s= %s %s' % (test_name.replace('/', ': '), results['avg'], unit))
155         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:]]))
156
157
158 class ChromiumStylePerfTest(PerfTest):
159     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
160
161     def __init__(self, port, test_name, path_or_url):
162         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
163
164     def parse_output(self, output):
165         test_failed = False
166         got_a_result = False
167         results = {}
168         for line in re.split('\n', output.text):
169             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
170             if resultLine:
171                 # FIXME: Store the unit
172                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
173                 _log.info(line)
174             elif not len(line) == 0:
175                 test_failed = True
176                 _log.error(line)
177         return results if results and not test_failed else None
178
179
180 class PageLoadingPerfTest(PerfTest):
181     def __init__(self, port, test_name, path_or_url):
182         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
183
184     def run(self, driver, time_out_ms):
185         test_times = []
186
187         for i in range(0, 20):
188             output = self.run_single(driver, self.path_or_url(), time_out_ms)
189             if not output or self.run_failed(output):
190                 return None
191             if i == 0:
192                 continue
193             test_times.append(output.test_time * 1000)
194
195         test_times = sorted(test_times)
196
197         # Compute the mean and variance using a numerically stable algorithm.
198         squareSum = 0
199         mean = 0
200         valueSum = sum(test_times)
201         for i, time in enumerate(test_times):
202             delta = time - mean
203             sweep = i + 1.0
204             mean += delta / sweep
205             squareSum += delta * delta * (i / sweep)
206
207         middle = int(len(test_times) / 2)
208         results = {'avg': mean,
209             'min': min(test_times),
210             'max': max(test_times),
211             'median': test_times[middle] if len(test_times) % 2 else (test_times[middle - 1] + test_times[middle]) / 2,
212             'stdev': math.sqrt(squareSum),
213             'unit': 'ms'}
214         self.output_statistics(self.test_name(), results, '')
215         return {self.test_name(): results}
216
217
218 class ReplayServer(object):
219     def __init__(self, archive, record):
220         self._process = None
221
222         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
223
224         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
225         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
226         if record:
227             args.append('--record')
228         args.append(archive)
229
230         self._process = subprocess.Popen(args)
231
232     def wait_until_ready(self):
233         for i in range(0, 10):
234             try:
235                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
236                 connection.close()
237                 return True
238             except socket.error:
239                 time.sleep(1)
240                 continue
241         return False
242
243     def stop(self):
244         if self._process:
245             self._process.send_signal(signal.SIGINT)
246             self._process.wait()
247         self._process = None
248
249     def __del__(self):
250         self.stop()
251
252
253 class ReplayPerfTest(PageLoadingPerfTest):
254     def __init__(self, port, test_name, path_or_url):
255         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
256
257     def _start_replay_server(self, archive, record):
258         try:
259             return ReplayServer(archive, record)
260         except OSError as error:
261             if error.errno == errno.ENOENT:
262                 _log.error("Replay tests require web-page-replay.")
263             else:
264                 raise error
265
266     def prepare(self, time_out_ms):
267         filesystem = self._port.host.filesystem
268         path_without_ext = filesystem.splitext(self.path_or_url())[0]
269
270         self._archive_path = filesystem.join(path_without_ext + '.wpr')
271         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
272         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
273
274         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
275             _log.info("Replay ready for %s" % self._archive_path)
276             return True
277
278         _log.info("Preparing replay for %s" % self.test_name())
279
280         driver = self._port.create_driver(worker_number=1, no_timeout=True)
281         try:
282             output = self.run_single(driver, self._url, time_out_ms, record=True)
283         finally:
284             driver.stop()
285
286         if not output or not filesystem.isfile(self._archive_path):
287             _log.error("Failed to prepare a replay for %s" % self.test_name())
288             return False
289
290         _log.info("Prepared replay for %s" % self.test_name())
291
292         return True
293
294     def run_single(self, driver, url, time_out_ms, record=False):
295         server = self._start_replay_server(self._archive_path, record)
296         if not server:
297             _log.error("Web page replay didn't start.")
298             return None
299
300         try:
301             if not server.wait_until_ready():
302                 _log.error("Web page replay didn't start.")
303                 return None
304
305             super(ReplayPerfTest, self).run_single(driver, "about:blank", time_out_ms)
306             _log.debug("Loading the page")
307
308             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
309             if self.run_failed(output):
310                 return None
311
312             if not output.image:
313                 _log.error("Loading the page did not generate image results")
314                 _log.error(output.text)
315                 return None
316
317             filesystem = self._port.host.filesystem
318             dirname = filesystem.dirname(url)
319             filename = filesystem.split(url)[1]
320             writer = TestResultWriter(filesystem, self._port, dirname, filename)
321             if record:
322                 writer.write_image_files(actual_image=None, expected_image=output.image)
323             else:
324                 writer.write_image_files(actual_image=output.image, expected_image=None)
325
326             return output
327         finally:
328             server.stop()
329
330
331 class PerfTestFactory(object):
332
333     _pattern_map = [
334         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
335         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
336         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
337     ]
338
339     @classmethod
340     def create_perf_test(cls, port, test_name, path):
341         for (pattern, test_class) in cls._pattern_map:
342             if pattern.match(test_name):
343                 return test_class(port, test_name, path)
344         return PerfTest(port, test_name, path)