Build fix after r125178.
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 import errno
32 import logging
33 import math
34 import re
35 import os
36 import signal
37 import socket
38 import subprocess
39 import sys
40 import time
41
42 # Import for auto-install
43 if sys.platform not in ('cygwin', 'win32'):
44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
46
47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
48 from webkitpy.layout_tests.port.driver import DriverInput
49 from webkitpy.layout_tests.port.driver import DriverOutput
50
51
52 _log = logging.getLogger(__name__)
53
54
55 class PerfTest(object):
56     def __init__(self, port, test_name, path_or_url):
57         self._port = port
58         self._test_name = test_name
59         self._path_or_url = path_or_url
60
61     def test_name(self):
62         return self._test_name
63
64     def path_or_url(self):
65         return self._path_or_url
66
67     def prepare(self, time_out_ms):
68         return True
69
70     def run(self, driver, time_out_ms):
71         output = self.run_single(driver, self.path_or_url(), time_out_ms)
72         if self.run_failed(output):
73             return None
74         return self.parse_output(output)
75
76     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
77         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test))
78
79     def run_failed(self, output):
80         if output.text == None or output.error:
81             pass
82         elif output.timeout:
83             _log.error('timeout: %s' % self.test_name())
84         elif output.crash:
85             _log.error('crash: %s' % self.test_name())
86         else:
87             return False
88
89         if output.error:
90             _log.error('error: %s\n%s' % (self.test_name(), output.error))
91
92         return True
93
94     _lines_to_ignore_in_parser_result = [
95         re.compile(r'^Running \d+ times$'),
96         re.compile(r'^Ignoring warm-up '),
97         re.compile(r'^Info:'),
98         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
99         # Following are for handle existing test like Dromaeo
100         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
101         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
102         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)""")),
103         # Following is for html5.html
104         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/"""))]
105
106     def _should_ignore_line_in_parser_test_result(self, line):
107         if not line:
108             return True
109         for regex in self._lines_to_ignore_in_parser_result:
110             if regex.search(line):
111                 return True
112         return False
113
114     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit']
115     _result_classes = ['Time:', 'JS Heap:', 'FastMalloc:']
116
117     def parse_output(self, output):
118         got_a_result = False
119         test_failed = False
120         results = dict([(name, dict()) for name in self._result_classes])
121         score_regex = re.compile(r'^(?P<key>' + r'|'.join(self._statistics_keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
122         description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
123         description_string = ""
124         result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(self._result_classes) + ')')
125
126         result_class = ""
127         for line in re.split('\n', output.text):
128             description = description_regex.match(line)
129             if description:
130                 description_string = description.group('description')
131                 continue
132
133             result_class_match = result_class_regex.match(line)
134             if result_class_match:
135                 result_class = result_class_match.group('resultclass')
136                 continue
137
138             score = score_regex.match(line)
139             if score:
140                 key = score.group('key')
141                 value = float(score.group('value'))
142                 unit = score.group('unit')
143                 results[result_class]['unit'] = unit
144                 results[result_class][key] = value
145                 continue
146
147             if not self._should_ignore_line_in_parser_test_result(line):
148                 test_failed = True
149                 _log.error(line)
150
151         if test_failed or set(self._statistics_keys) != set(results[self._result_classes[0]].keys()):
152             return None
153
154         test_name = re.sub(r'\.\w+$', '', self._test_name)
155         self.output_statistics(test_name, results[self._result_classes[0]], description_string)
156         if results[self._result_classes[1]] and results[self._result_classes[2]]:
157             self.output_statistics(test_name + "/JSHeap", results[self._result_classes[1]])
158             self.output_statistics(test_name + "/FastMalloc", results[self._result_classes[2]])
159         return {test_name: results['Time:']}
160
161     def output_statistics(self, test_name, results, description_string=None):
162         unit = results['unit']
163         if description_string:
164             _log.info('DESCRIPTION: %s' % description_string)
165         _log.info('RESULT %s= %s %s' % (test_name.replace('/', ': '), results['avg'], unit))
166         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
167
168
169 class ChromiumStylePerfTest(PerfTest):
170     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
171
172     def __init__(self, port, test_name, path_or_url):
173         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
174
175     def parse_output(self, output):
176         test_failed = False
177         got_a_result = False
178         results = {}
179         for line in re.split('\n', output.text):
180             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
181             if resultLine:
182                 # FIXME: Store the unit
183                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
184                 _log.info(line)
185             elif not len(line) == 0:
186                 test_failed = True
187                 _log.error(line)
188         return results if results and not test_failed else None
189
190
191 class PageLoadingPerfTest(PerfTest):
192     def __init__(self, port, test_name, path_or_url):
193         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
194
195     def run(self, driver, time_out_ms):
196         test_times = []
197
198         for i in range(0, 20):
199             output = self.run_single(driver, self.path_or_url(), time_out_ms)
200             if not output or self.run_failed(output):
201                 return None
202             if i == 0:
203                 continue
204             test_times.append(output.test_time * 1000)
205
206         test_times = sorted(test_times)
207
208         # Compute the mean and variance using a numerically stable algorithm.
209         squareSum = 0
210         mean = 0
211         valueSum = sum(test_times)
212         for i, time in enumerate(test_times):
213             delta = time - mean
214             sweep = i + 1.0
215             mean += delta / sweep
216             squareSum += delta * delta * (i / sweep)
217
218         middle = int(len(test_times) / 2)
219         results = {'avg': mean,
220             'min': min(test_times),
221             'max': max(test_times),
222             'median': test_times[middle] if len(test_times) % 2 else (test_times[middle - 1] + test_times[middle]) / 2,
223             'stdev': math.sqrt(squareSum),
224             'unit': 'ms'}
225         self.output_statistics(self.test_name(), results, '')
226         return {self.test_name(): results}
227
228
229 class ReplayServer(object):
230     def __init__(self, archive, record):
231         self._process = None
232
233         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
234
235         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
236         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
237         if record:
238             args.append('--record')
239         args.append(archive)
240
241         self._process = subprocess.Popen(args)
242
243     def wait_until_ready(self):
244         for i in range(0, 3):
245             try:
246                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
247                 connection.close()
248                 return True
249             except socket.error:
250                 time.sleep(1)
251                 continue
252         return False
253
254     def stop(self):
255         if self._process:
256             self._process.send_signal(signal.SIGINT)
257             self._process.wait()
258         self._process = None
259
260     def __del__(self):
261         self.stop()
262
263
264 class ReplayPerfTest(PageLoadingPerfTest):
265     def __init__(self, port, test_name, path_or_url):
266         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
267
268     def _start_replay_server(self, archive, record):
269         try:
270             return ReplayServer(archive, record)
271         except OSError as error:
272             if error.errno == errno.ENOENT:
273                 _log.error("Replay tests require web-page-replay.")
274             else:
275                 raise error
276
277     def prepare(self, time_out_ms):
278         filesystem = self._port.host.filesystem
279         path_without_ext = filesystem.splitext(self.path_or_url())[0]
280
281         self._archive_path = filesystem.join(path_without_ext + '.wpr')
282         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
283         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
284
285         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
286             _log.info("Replay ready for %s" % self._archive_path)
287             return True
288
289         _log.info("Preparing replay for %s" % self.test_name())
290
291         driver = self._port.create_driver(worker_number=1, no_timeout=True)
292         try:
293             output = self.run_single(driver, self._archive_path, time_out_ms, record=True)
294         finally:
295             driver.stop()
296
297         if not output or not filesystem.isfile(self._archive_path):
298             _log.error("Failed to prepare a replay for %s" % self.test_name())
299             return False
300
301         _log.info("Prepared replay for %s" % self.test_name())
302
303         return True
304
305     def run_single(self, driver, url, time_out_ms, record=False):
306         server = self._start_replay_server(self._archive_path, record)
307         if not server:
308             _log.error("Web page replay didn't start.")
309             return None
310
311         try:
312             _log.debug("Waiting for Web page replay to start.")
313             if not server.wait_until_ready():
314                 _log.error("Web page replay didn't start.")
315                 return None
316
317             super(ReplayPerfTest, self).run_single(driver, "about:blank", time_out_ms)
318             _log.debug("Loading the page")
319
320             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
321             if self.run_failed(output):
322                 return None
323
324             if not output.image:
325                 _log.error("Loading the page did not generate image results")
326                 _log.error(output.text)
327                 return None
328
329             filesystem = self._port.host.filesystem
330             dirname = filesystem.dirname(self._archive_path)
331             filename = filesystem.split(self._archive_path)[1]
332             writer = TestResultWriter(filesystem, self._port, dirname, filename)
333             if record:
334                 writer.write_image_files(actual_image=None, expected_image=output.image)
335             else:
336                 writer.write_image_files(actual_image=output.image, expected_image=None)
337
338             return output
339         finally:
340             server.stop()
341
342
343 class PerfTestFactory(object):
344
345     _pattern_map = [
346         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
347         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
348         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
349     ]
350
351     @classmethod
352     def create_perf_test(cls, port, test_name, path):
353         for (pattern, test_class) in cls._pattern_map:
354             if pattern.match(test_name):
355                 return test_class(port, test_name, path)
356         return PerfTest(port, test_name, path)