53a5a3b728364955c0565816916332999c9643ed
[WebKit-https.git] / PerformanceTests / LaunchTime / launch_time.py
1 import SimpleHTTPServer
2 import SocketServer
3 import argparse
4 import logging
5 from math import sqrt
6 from operator import mul
7 import os
8 from subprocess import call, check_output
9 import sys
10 import threading
11 import time
12
13 from feedback_server import FeedbackServer
14
15
16 # Supress logs from feedback server
17 logging.getLogger().setLevel(logging.FATAL)
18
19
20 class DefaultLaunchTimeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
21     def get_test_page(self):
22         return '''<!DOCTYPE html>
23         <html>
24           <head>
25             <title>Launch Time Benchmark</title>
26             <meta http-equiv="Content-Type" content="text/html" />
27             <script>
28                 function sendDone() {
29                     const time = performance.timing.navigationStart
30                     const request = new XMLHttpRequest();
31                     request.open("POST", "done", false);
32                     request.setRequestHeader('Content-Type', 'application/json');
33                     request.send(JSON.stringify(time));
34                 }
35                 window.onload = sendDone;
36             </script>
37           </head>
38           <body>
39             <h1>New Tab Benchmark</h1>
40           </body>
41         </html>
42         '''
43
44     def on_receive_stop_signal(self, data):
45         pass
46
47     def do_HEAD(self):
48         self.send_response(200)
49         self.send_header('Content-type', 'text/html')
50         self.end_headers()
51
52     def do_GET(self):
53         self.send_response(200)
54         self.send_header('Content-type', 'text/html')
55         self.end_headers()
56         if not self.path.startswith('/blank'):
57             self.wfile.write(self.get_test_page())
58         self.wfile.close()
59
60     def do_POST(self):
61         self.send_response(200)
62         self.send_header('Content-type', 'text/html')
63         self.end_headers()
64         self.wfile.write('done')
65         self.wfile.close()
66
67         data_string = self.rfile.read(int(self.headers['Content-Length']))
68         self.on_receive_stop_signal(data_string)
69
70     def log_message(self, format, *args):
71         pass
72
73
74 class LaunchTimeBenchmark:
75     def __init__(self):
76         self._server_ready = threading.Semaphore(0)
77         self._server = None
78         self._server_thread = None
79         self._port = 8080
80         self._feedback_port = None
81         self._feedback_server = None
82         self._open_count = 0
83         self._app_name = None
84         self._verbose = False
85         self._feedback_in_browser = False
86         self._do_not_ignore_first_result = False
87         self._iterations = 5
88         self._browser_bundle_path = '/Applications/Safari.app'
89         self.response_handler = None
90         self.benchmark_description = None
91         self.use_geometric_mean = False
92         self.wait_time_high = 1
93         self.wait_time_low = 0.1
94         self.iteration_groups = 1
95         self.initialize()
96
97     def _parse_browser_bundle_path(self, path):
98         if not os.path.isdir(path) or not path.endswith('.app'):
99             raise argparse.ArgumentTypeError(
100                 'Invalid app bundle path: "{}"'.format(path))
101         return path
102
103     def _parse_args(self):
104         self.argument_parser = argparse.ArgumentParser(description=self.benchmark_description)
105         self.argument_parser.add_argument('-p', '--path', type=self._parse_browser_bundle_path,
106             help='path for browser application bundle (default: {})'.format(self._browser_bundle_path))
107         self.argument_parser.add_argument('-n', '--iterations', type=int,
108             help='number of iterations of test (default: {})'.format(self._iterations))
109         self.argument_parser.add_argument('-v', '--verbose', action='store_true',
110             help="print each iteration's time")
111         self.argument_parser.add_argument('-f', '--feedback-in-browser', action='store_true',
112             help="show benchmark results in browser (default: {})".format(self._feedback_in_browser))
113         self.will_parse_arguments()
114
115         args = self.argument_parser.parse_args()
116         if args.iterations:
117             self._iterations = args.iterations
118         if args.path:
119             self._browser_bundle_path = args.path
120         if args.verbose is not None:
121             self._verbose = args.verbose
122         if args.feedback_in_browser is not None:
123             self._feedback_in_browser = args.feedback_in_browser
124         path_len = len(self._browser_bundle_path)
125         start_index = self._browser_bundle_path.rfind('/', 0, path_len)
126         end_index = self._browser_bundle_path.rfind('.', 0, path_len)
127         self._app_name = self._browser_bundle_path[start_index + 1:end_index]
128         self.did_parse_arguments(args)
129
130     def _run_server(self):
131         self._server_ready.release()
132         self._server.serve_forever()
133
134     def _setup_servers(self):
135         while True:
136             try:
137                 self._server = SocketServer.TCPServer(
138                     ('0.0.0.0', self._port), self.response_handler)
139                 break
140             except:
141                 self._port += 1
142         print 'Running test server at http://localhost:{}'.format(self._port)
143
144         self._server_thread = threading.Thread(target=self._run_server)
145         self._server_thread.start()
146         self._server_ready.acquire()
147
148         if self._feedback_in_browser:
149             self._feedback_server = FeedbackServer()
150             self._feedback_port = self._feedback_server.start()
151
152     def _clean_up(self):
153         self._server.shutdown()
154         self._server_thread.join()
155         if self._feedback_in_browser:
156             self._feedback_server.stop()
157
158     def _exit_due_to_exception(self, reason):
159         self.log(reason)
160         self._clean_up()
161         sys.exit(1)
162
163     def _geometric_mean(self, values):
164         product = reduce(mul, values)
165         return product ** (1.0 / len(values))
166
167     def _standard_deviation(self, results, mean=None):
168         if mean is None:
169             mean = sum(results) / float(len(results))
170         divisor = float(len(results) - 1) if len(results) > 1 else float(len(results))
171         variance = sum((x - mean) ** 2 for x in results) / divisor
172         return sqrt(variance)
173
174     def _compute_results(self, results):
175         if not results:
176             self._exit_due_to_exception('No results to compute.\n')
177         if len(results) > 1 and not self._do_not_ignore_first_result:
178             results = results[1:]
179         mean = sum(results) / float(len(results))
180         stdev = self._standard_deviation(results, mean)
181         return mean, stdev
182
183     def _wait_times(self):
184         if self.iteration_groups == 1:
185             yield self.wait_time_high
186             return
187         increment_per_group = float(self.wait_time_high - self.wait_time_low) / (self.iteration_groups - 1)
188         for i in range(self.iteration_groups):
189             yield self.wait_time_low + increment_per_group * i
190
191     def open_tab(self, blank=False):
192         if blank:
193             call(['open', '-a', self._browser_bundle_path,
194                 'http://localhost:{}/blank/{}'.format(self._port, self._open_count)])
195         else:
196             call(['open', '-a', self._browser_bundle_path,
197                 'http://localhost:{}/{}'.format(self._port, self._open_count)])
198         self._open_count += 1
199
200     def launch_browser(self):
201         if self._feedback_in_browser:
202             call(['open', '-a', self._browser_bundle_path,
203                 'http://localhost:{}'.format(self._feedback_port), '-F'])
204             self._feedback_server.wait_until_client_has_loaded()
205         else:
206             call(['open', '-a', self._browser_bundle_path,
207                 'http://localhost:{}/blank'.format(self._port), '-F'])
208         self.wait(2)
209
210     def quit_browser(self):
211         def quit_app():
212             call(['osascript', '-e', 'quit app "{}"'.format(self._browser_bundle_path)])
213
214         def is_app_closed():
215             out = check_output(['osascript', '-e', 'tell application "System Events"',
216                 '-e', 'copy (get name of every process whose name is "{}") to stdout'.format(self._app_name),
217                 '-e', 'end tell'])
218             return len(out.strip()) == 0
219
220         while not is_app_closed():
221             quit_app()
222         self.wait(1)
223
224     def close_tab(self):
225         call(['osascript', '-e',
226             'tell application "System Events" to keystroke "w" using command down'])
227
228     def wait(self, duration):
229         wait_start = time.time()
230         while time.time() - wait_start < duration:
231             pass
232
233     def log(self, message):
234         if self._feedback_in_browser:
235             self._feedback_server.send_message(message)
236         sys.stdout.write(message)
237         sys.stdout.flush()
238
239     def log_verbose(self, message):
240         if self._verbose:
241             self.log(message)
242
243     def run(self):
244         self._parse_args()
245         self._setup_servers()
246         self.quit_browser()
247         print ''
248
249         try:
250             group_means = []
251             results_by_iteration_number = [[] for _ in range(self._iterations)]
252
253             group = 1
254             for wait_duration in self._wait_times():
255                 self.group_init()
256                 if self.iteration_groups > 1:
257                     self.log('Running group {}{}'.format(group, ':\n' if self._verbose else '...'))
258
259                 results = []
260                 for i in range(self._iterations):
261                     try:
262                         if not self._verbose:
263                             self.log('.')
264                         result_in_ms = self.run_iteration()
265                         self.log_verbose('({}) {} ms\n'.format(i + 1, result_in_ms))
266                         self.wait(wait_duration)
267                         results.append(result_in_ms)
268                         results_by_iteration_number[i].append(result_in_ms)
269                     except KeyboardInterrupt:
270                         raise KeyboardInterrupt
271                     except Exception as error:
272                         self._exit_due_to_exception('(Test {} failed) {}: {}\n'.format(i + 1 if self._verbose else i, type(error).__name__, error))
273                 if not self._verbose:
274                     print ''
275
276                 mean, stdev = self._compute_results(results)
277                 self.log_verbose('RESULTS:\n')
278                 self.log_verbose('mean: {} ms\n'.format(mean))
279                 self.log_verbose('std dev: {} ms ({}%)\n\n'.format(stdev, (stdev / mean) * 100))
280                 if self._verbose:
281                     self.wait(1)
282                 group_means.append(mean)
283                 group += 1
284                 self.quit_browser()
285
286             if not self._verbose:
287                 print '\n'
288
289             if self._feedback_in_browser:
290                 self.launch_browser()
291
292             means_by_iteration_number = []
293             if len(results_by_iteration_number) > 1 and not self._do_not_ignore_first_result:
294                 results_by_iteration_number = results_by_iteration_number[1:]
295             for iteration_results in results_by_iteration_number:
296                 means_by_iteration_number.append(self._geometric_mean(iteration_results))
297             final_mean = self._geometric_mean(group_means)
298             final_stdev = self._standard_deviation(means_by_iteration_number)
299             self.log('FINAL RESULTS\n')
300             self.log('Mean:\n-> {} ms\n'.format(final_mean))
301             self.log('Standard Deviation:\n-> {} ms ({}%)\n'.format(final_stdev, (final_stdev / final_mean) * 100))
302         except KeyboardInterrupt:
303             self._clean_up()
304             sys.exit(1)
305         finally:
306             self._clean_up()
307
308     def group_init(self):
309         pass
310
311     def run_iteration(self):
312         pass
313
314     def initialize(self):
315         pass
316
317     def will_parse_arguments(self):
318         pass
319
320     def did_parse_arguments(self, args):
321         pass