Add benchmark for WebKit process launch times
[WebKit-https.git] / PerformanceTests / LaunchTime / launch_time.py
1 import SimpleHTTPServer
2 import SocketServer
3 import argparse
4 import logging
5 from math import sqrt
6 from operator import mul
7 import os
8 from subprocess import call, check_output
9 import sys
10 import threading
11 import time
12
13 from feedback_server import FeedbackServer
14
15
16 # Supress logs from feedback server
17 logging.getLogger().setLevel(logging.FATAL)
18
19
20 class DefaultLaunchTimeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
21     def get_test_page(self):
22         return '''<!DOCTYPE html>
23         <html>
24           <head>
25             <title>Launch Time Benchmark</title>
26             <meta http-equiv="Content-Type" content="text/html" />
27             <script>
28                 function sendDone() {
29                     const time = performance.timing.navigationStart
30                     const request = new XMLHttpRequest();
31                     request.open("POST", "done", false);
32                     request.setRequestHeader('Content-Type', 'application/json');
33                     request.send(JSON.stringify(time));
34                 }
35                 window.onload = sendDone;
36             </script>
37           </head>
38           <body>
39             <h1>New Tab Benchmark</h1>
40           </body>
41         </html>
42         '''
43
44     def get_blank_page(self):
45         return '''<!DOCTYPE html>
46         <html>
47           <head>
48             <title>Launch Time Benchmark</title>
49             <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
50           </head>
51         </html>'''
52
53     def on_receive_stop_time(self, time):
54         pass
55
56     def do_HEAD(self):
57         self.send_response(200)
58         self.send_header('Content-type', 'text/hetml')
59         self.end_headers()
60
61     def do_GET(self):
62         self.send_response(200)
63         self.send_header('Content-type', 'text/html')
64         self.end_headers()
65         self.wfile.write(self.get_blank_page() if self.path == '/blank' else self.get_test_page())
66         self.wfile.close()
67
68     def do_POST(self):
69         self.send_response(200)
70         self.send_header('Content-type', 'text/html')
71         self.end_headers()
72         self.wfile.write('done')
73         self.wfile.close()
74
75         data_string = self.rfile.read(int(self.headers['Content-Length']))
76         time = float(data_string)
77
78         self.on_receive_stop_time(time)
79
80     def log_message(self, format, *args):
81         pass
82
83
84 class LaunchTimeBenchmark:
85     def __init__(self):
86         self._server_ready = threading.Semaphore(0)
87         self._server = None
88         self._server_thread = None
89         self._port = 8080
90         self._feedback_port = None
91         self._feedback_server = None
92         self._open_count = 0
93         self._app_name = None
94         self._verbose = False
95         self._feedback_in_browser = False
96         self._do_not_ignore_first_result = False
97         self._iterations = 5
98         self._browser_bundle_path = '/Applications/Safari.app'
99         self.response_handler = None
100         self.benchmark_description = None
101         self.use_geometric_mean = False
102         self.wait_time_high = 1
103         self.wait_time_low = 0.1
104         self.iteration_groups = 1
105         self.initialize()
106
107     def _parse_browser_bundle_path(self, path):
108         if not os.path.isdir(path) or not path.endswith('.app'):
109             raise argparse.ArgumentTypeError(
110                 'Invalid app bundle path: "{}"'.format(path))
111         return path
112
113     def _parse_args(self):
114         self.argument_parser = argparse.ArgumentParser(description=self.benchmark_description)
115         self.argument_parser.add_argument('-p', '--path', type=self._parse_browser_bundle_path,
116             help='path for browser application bundle (default: {})'.format(self._browser_bundle_path))
117         self.argument_parser.add_argument('-n', '--iterations', type=int,
118             help='number of iterations of test (default: {})'.format(self._iterations))
119         self.argument_parser.add_argument('-v', '--verbose', action='store_true',
120             help="print each iteration's time")
121         self.argument_parser.add_argument('-f', '--feedback-in-browser', action='store_true',
122             help="show benchmark results in browser (default: {})".format(self._feedback_in_browser))
123         self.will_parse_arguments()
124
125         args = self.argument_parser.parse_args()
126         if args.iterations:
127             self._iterations = args.iterations
128         if args.path:
129             self._browser_bundle_path = args.path
130         if args.verbose is not None:
131             self._verbose = args.verbose
132         if args.feedback_in_browser is not None:
133             self._feedback_in_browser = args.feedback_in_browser
134         path_len = len(self._browser_bundle_path)
135         start_index = self._browser_bundle_path.rfind('/', 0, path_len)
136         end_index = self._browser_bundle_path.rfind('.', 0, path_len)
137         self._app_name = self._browser_bundle_path[start_index + 1:end_index]
138         self.did_parse_arguments(args)
139
140     def _run_server(self):
141         self._server_ready.release()
142         self._server.serve_forever()
143
144     def _setup_servers(self):
145         while True:
146             try:
147                 self._server = SocketServer.TCPServer(
148                     ('0.0.0.0', self._port), self.response_handler)
149                 break
150             except:
151                 self._port += 1
152         print 'Running test server at http://localhost:{}'.format(self._port)
153
154         self._server_thread = threading.Thread(target=self._run_server)
155         self._server_thread.start()
156         self._server_ready.acquire()
157
158         if self._feedback_in_browser:
159             self._feedback_server = FeedbackServer()
160             self._feedback_port = self._feedback_server.start()
161
162     def _clean_up(self):
163         self._server.shutdown()
164         self._server_thread.join()
165         if self._feedback_in_browser:
166             self._feedback_server.stop()
167
168     def _exit_due_to_exception(self, reason):
169         self.log(reason)
170         self._clean_up()
171         sys.exit(1)
172
173     def _geometric_mean(self, values):
174         product = reduce(mul, values)
175         return product ** (1.0 / len(values))
176
177     def _standard_deviation(self, results, mean=None):
178         if mean is None:
179             mean = sum(results) / float(len(results))
180         variance = sum((x - mean) ** 2 for x in results) / float(len(results) - 1)
181         return sqrt(variance)
182
183     def _compute_results(self, results):
184         if not results:
185             self._exit_due_to_exception('No results to compute.\n')
186         if len(results) > 1 and not self._do_not_ignore_first_result:
187             results = results[1:]
188         mean = sum(results) / float(len(results))
189         stdev = self._standard_deviation(results, mean)
190         return mean, stdev
191
192     def _wait_times(self):
193         if self.iteration_groups == 1:
194             yield self.wait_time_high
195             return
196         increment_per_group = float(self.wait_time_high - self.wait_time_low) / (self.iteration_groups - 1)
197         for i in range(self.iteration_groups):
198             yield self.wait_time_low + increment_per_group * i
199
200     def open_tab(self):
201         call(['open', '-a', self._browser_bundle_path,
202             'http://localhost:{}/{}'.format(self._port, self._open_count)])
203         self._open_count += 1
204
205     def launch_browser(self):
206         if self._feedback_in_browser:
207             call(['open', '-a', self._browser_bundle_path,
208                 'http://localhost:{}'.format(self._feedback_port), '-F'])
209             self._feedback_server.wait_until_client_has_loaded()
210         else:
211             call(['open', '-a', self._browser_bundle_path,
212                 'http://localhost:{}/blank'.format(self._port), '-F'])
213         self.wait(2)
214
215     def quit_browser(self):
216         def quit_app():
217             call(['osascript', '-e', 'quit app "{}"'.format(self._browser_bundle_path)])
218
219         def is_app_closed():
220             out = check_output(['osascript', '-e', 'tell application "System Events"',
221                 '-e', 'copy (get name of every process whose name is "{}") to stdout'.format(self._app_name),
222                 '-e', 'end tell'])
223             return len(out.strip()) == 0
224
225         while not is_app_closed():
226             quit_app()
227         self.wait(1)
228
229     def close_tab(self):
230         call(['osascript', '-e',
231             'tell application "System Events" to keystroke "w" using command down'])
232
233     def wait(self, duration):
234         wait_start = time.time()
235         while time.time() - wait_start < duration:
236             pass
237
238     def log(self, message):
239         if self._feedback_in_browser:
240             self._feedback_server.send_message(message)
241         sys.stdout.write(message)
242         sys.stdout.flush()
243
244     def log_verbose(self, message):
245         if self._verbose:
246             self.log(message)
247
248     def run(self):
249         self._parse_args()
250         self._setup_servers()
251         self.quit_browser()
252         print ''
253
254         try:
255             group_means = []
256             results_by_iteration_number = [[] for _ in range(self._iterations)]
257
258             group = 1
259             for wait_duration in self._wait_times():
260                 self.group_init()
261                 if self.iteration_groups > 1:
262                     self.log('Running group {}{}'.format(group, ':\n' if self._verbose else '...'))
263
264                 results = []
265                 for i in range(self._iterations):
266                     try:
267                         if not self._verbose:
268                             self.log('.')
269                         result_in_ms = self.run_iteration()
270                         self.log_verbose('({}) {} ms\n'.format(i + 1, result_in_ms))
271                         self.wait(wait_duration)
272                         results.append(result_in_ms)
273                         results_by_iteration_number[i].append(result_in_ms)
274                     except KeyboardInterrupt:
275                         raise KeyboardInterrupt
276                     except:
277                         self._exit_due_to_exception('(Test {} failed)\n'.format(i + 1 if self._verbose else i))
278                 if not self._verbose:
279                     print ''
280
281                 mean, stdev = self._compute_results(results)
282                 self.log_verbose('RESULTS:\n')
283                 self.log_verbose('mean: {} ms\n'.format(mean))
284                 self.log_verbose('std dev: {} ms ({}%)\n\n'.format(stdev, (stdev / mean) * 100))
285                 if self._verbose:
286                     self.wait(1)
287                 group_means.append(mean)
288                 group += 1
289                 self.quit_browser()
290
291             if not self._verbose:
292                 print '\n'
293
294             if self._feedback_in_browser:
295                 self.launch_browser()
296
297             means_by_iteration_number = []
298             if len(results_by_iteration_number) > 1 and not self._do_not_ignore_first_result:
299                 results_by_iteration_number = results_by_iteration_number[1:]
300             for iteration_results in results_by_iteration_number:
301                 means_by_iteration_number.append(self._geometric_mean(iteration_results))
302             final_mean = self._geometric_mean(group_means)
303             final_stdev = self._standard_deviation(means_by_iteration_number)
304             self.log('FINAL RESULTS\n')
305             self.log('Mean:\n-> {} ms\n'.format(final_mean))
306             self.log('Standard Deviation:\n-> {} ms ({}%)\n'.format(final_stdev, (final_stdev / final_mean) * 100))
307         except KeyboardInterrupt:
308             self._clean_up()
309             sys.exit(1)
310         finally:
311             self._clean_up()
312
313     def group_init(self):
314         pass
315
316     def run_iteration(self):
317         pass
318
319     def initialize(self):
320         pass
321
322     def will_parse_arguments(self):
323         pass
324
325     def did_parse_arguments(self, args):
326         pass