Import FeedbackServer only if "-f/--feedback-in-browser" option is enabled.
[WebKit-https.git] / PerformanceTests / LaunchTime / launch_time.py
1 import SimpleHTTPServer
2 import SocketServer
3 import argparse
4 import logging
5 from math import sqrt
6 from operator import mul
7 import os
8 from subprocess import call, check_output
9 import sys
10 import threading
11 import time
12
13
14
15 # Supress logs from feedback server
16 logging.getLogger().setLevel(logging.FATAL)
17
18
19 class DefaultLaunchTimeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
20     def get_test_page(self):
21         return '''<!DOCTYPE html>
22         <html>
23           <head>
24             <title>Launch Time Benchmark</title>
25             <meta http-equiv="Content-Type" content="text/html" />
26             <script>
27                 function sendDone() {
28                     const time = performance.timing.navigationStart
29                     const request = new XMLHttpRequest();
30                     request.open("POST", "done", false);
31                     request.setRequestHeader('Content-Type', 'application/json');
32                     request.send(JSON.stringify(time));
33                 }
34                 window.onload = sendDone;
35             </script>
36           </head>
37           <body>
38             <h1>New Tab Benchmark</h1>
39           </body>
40         </html>
41         '''
42
43     def on_receive_stop_signal(self, data):
44         pass
45
46     def do_HEAD(self):
47         self.send_response(200)
48         self.send_header('Content-type', 'text/html')
49         self.end_headers()
50
51     def do_GET(self):
52         self.send_response(200)
53         self.send_header('Content-type', 'text/html')
54         self.end_headers()
55         if not self.path.startswith('/blank'):
56             self.wfile.write(self.get_test_page())
57         self.wfile.close()
58
59     def do_POST(self):
60         self.send_response(200)
61         self.send_header('Content-type', 'text/html')
62         self.end_headers()
63         self.wfile.write('done')
64         self.wfile.close()
65
66         data_string = self.rfile.read(int(self.headers['Content-Length']))
67         self.on_receive_stop_signal(data_string)
68
69     def log_message(self, format, *args):
70         pass
71
72
73 class LaunchTimeBenchmark:
74     def __init__(self):
75         self._server_ready = threading.Semaphore(0)
76         self._server = None
77         self._server_thread = None
78         self._port = 8080
79         self._feedback_port = None
80         self._feedback_server = None
81         self._open_count = 0
82         self._app_name = None
83         self._verbose = False
84         self._feedback_in_browser = False
85         self._do_not_ignore_first_result = False
86         self._iterations = 5
87         self._browser_bundle_path = '/Applications/Safari.app'
88         self.response_handler = None
89         self.benchmark_description = None
90         self.use_geometric_mean = False
91         self.wait_time_high = 1
92         self.wait_time_low = 0.1
93         self.iteration_groups = 1
94         self.initialize()
95
96     def _parse_browser_bundle_path(self, path):
97         if not os.path.isdir(path) or not path.endswith('.app'):
98             raise argparse.ArgumentTypeError(
99                 'Invalid app bundle path: "{}"'.format(path))
100         return path
101
102     def _parse_args(self):
103         self.argument_parser = argparse.ArgumentParser(description=self.benchmark_description)
104         self.argument_parser.add_argument('-p', '--path', type=self._parse_browser_bundle_path,
105             help='path for browser application bundle (default: {})'.format(self._browser_bundle_path))
106         self.argument_parser.add_argument('-n', '--iterations', type=int,
107             help='number of iterations of test (default: {})'.format(self._iterations))
108         self.argument_parser.add_argument('-v', '--verbose', action='store_true',
109             help="print each iteration's time")
110         self.argument_parser.add_argument('-f', '--feedback-in-browser', action='store_true',
111             help="show benchmark results in browser (default: {})".format(self._feedback_in_browser))
112         self.will_parse_arguments()
113
114         args = self.argument_parser.parse_args()
115         if args.iterations:
116             self._iterations = args.iterations
117         if args.path:
118             self._browser_bundle_path = args.path
119         if args.verbose is not None:
120             self._verbose = args.verbose
121         if args.feedback_in_browser is not None:
122             self._feedback_in_browser = args.feedback_in_browser
123         path_len = len(self._browser_bundle_path)
124         start_index = self._browser_bundle_path.rfind('/', 0, path_len)
125         end_index = self._browser_bundle_path.rfind('.', 0, path_len)
126         self._app_name = self._browser_bundle_path[start_index + 1:end_index]
127         self.did_parse_arguments(args)
128
129     def _run_server(self):
130         self._server_ready.release()
131         self._server.serve_forever()
132
133     def _setup_servers(self):
134         while True:
135             try:
136                 self._server = SocketServer.TCPServer(
137                     ('0.0.0.0', self._port), self.response_handler)
138                 break
139             except:
140                 self._port += 1
141         print 'Running test server at http://localhost:{}'.format(self._port)
142
143         self._server_thread = threading.Thread(target=self._run_server)
144         self._server_thread.start()
145         self._server_ready.acquire()
146
147         if self._feedback_in_browser:
148             from feedback_server import FeedbackServer
149             self._feedback_server = FeedbackServer()
150             self._feedback_port = self._feedback_server.start()
151
152     def _clean_up(self):
153         self._server.shutdown()
154         self._server_thread.join()
155         if self._feedback_in_browser:
156             self._feedback_server.stop()
157
158     def _exit_due_to_exception(self, reason):
159         self.log(reason)
160         self._clean_up()
161         sys.exit(1)
162
163     def _geometric_mean(self, values):
164         product = reduce(mul, values)
165         return product ** (1.0 / len(values))
166
167     def _standard_deviation(self, results, mean=None):
168         if mean is None:
169             mean = sum(results) / float(len(results))
170         divisor = float(len(results) - 1) if len(results) > 1 else float(len(results))
171         variance = sum((x - mean) ** 2 for x in results) / divisor
172         return sqrt(variance)
173
174     def _compute_results(self, results):
175         if not results:
176             self._exit_due_to_exception('No results to compute.\n')
177         if len(results) > 1 and not self._do_not_ignore_first_result:
178             results = results[1:]
179         mean = sum(results) / float(len(results))
180         stdev = self._standard_deviation(results, mean)
181         return mean, stdev
182
183     def _wait_times(self):
184         if self.iteration_groups == 1:
185             yield self.wait_time_high
186             return
187         increment_per_group = float(self.wait_time_high - self.wait_time_low) / (self.iteration_groups - 1)
188         for i in range(self.iteration_groups):
189             yield self.wait_time_low + increment_per_group * i
190
191     def open_tab(self, blank=False):
192         if blank:
193             call(['open', '-a', self._browser_bundle_path,
194                 'http://localhost:{}/blank/{}'.format(self._port, self._open_count)])
195         else:
196             call(['open', '-a', self._browser_bundle_path,
197                 'http://localhost:{}/{}'.format(self._port, self._open_count)])
198         self._open_count += 1
199
200     def launch_browser(self):
201         if self._feedback_in_browser:
202             call(['open', '-a', self._browser_bundle_path,
203                 'http://localhost:{}'.format(self._feedback_port), '-F'])
204             self._feedback_server.wait_until_client_has_loaded()
205         else:
206             call(['open', '-a', self._browser_bundle_path,
207                 'http://localhost:{}/blank'.format(self._port), '-F'])
208         self.wait(2)
209
210     def quit_browser(self):
211         def quit_app():
212             call(['osascript', '-e', 'quit app "{}"'.format(self._browser_bundle_path)])
213
214         def is_app_closed():
215             out = check_output(['osascript', '-e', 'tell application "System Events"',
216                 '-e', 'copy (get name of every process whose name is "{}") to stdout'.format(self._app_name),
217                 '-e', 'end tell'])
218             return len(out.strip()) == 0
219
220         while not is_app_closed():
221             quit_app()
222         self.wait(1)
223
224     def close_tab(self):
225         call(['osascript', '-e',
226             'tell application "System Events" to keystroke "w" using command down'])
227
228     def wait(self, duration):
229         wait_start = time.time()
230         while time.time() - wait_start < duration:
231             pass
232
233     def log(self, message):
234         if self._feedback_in_browser:
235             self._feedback_server.send_message(message)
236         sys.stdout.write(message)
237         sys.stdout.flush()
238
239     def log_verbose(self, message):
240         if self._verbose:
241             self.log(message)
242
243     def run(self):
244         self._parse_args()
245         self._setup_servers()
246         self.quit_browser()
247         print ''
248
249         try:
250             group_means = []
251             results_by_iteration_number = [[] for _ in range(self._iterations)]
252
253             group = 1
254             for wait_duration in self._wait_times():
255                 self.group_init()
256                 if self.iteration_groups > 1:
257                     self.log('Running group {}{}'.format(group, ':\n' if self._verbose else '...'))
258
259                 results = []
260                 for i in range(self._iterations):
261                     try:
262                         if not self._verbose:
263                             self.log('.')
264                         result_in_ms = self.run_iteration()
265                         self.log_verbose('({}) {} ms\n'.format(i + 1, result_in_ms))
266                         self.wait(wait_duration)
267                         results.append(result_in_ms)
268                         results_by_iteration_number[i].append(result_in_ms)
269                     except KeyboardInterrupt:
270                         raise KeyboardInterrupt
271                     except Exception as error:
272                         self._exit_due_to_exception('(Test {} failed) {}: {}\n'.format(i + 1 if self._verbose else i, type(error).__name__, error))
273                 if not self._verbose:
274                     print ''
275
276                 mean, stdev = self._compute_results(results)
277                 self.log_verbose('RESULTS:\n')
278                 self.log_verbose('mean: {} ms\n'.format(mean))
279                 self.log_verbose('std dev: {} ms ({}%)\n\n'.format(stdev, (stdev / mean) * 100))
280                 if self._verbose:
281                     self.wait(1)
282                 group_means.append(mean)
283                 group += 1
284                 self.quit_browser()
285
286             if not self._verbose:
287                 print '\n'
288
289             if self._feedback_in_browser:
290                 self.launch_browser()
291
292             means_by_iteration_number = []
293             if len(results_by_iteration_number) > 1 and not self._do_not_ignore_first_result:
294                 results_by_iteration_number = results_by_iteration_number[1:]
295             for iteration_results in results_by_iteration_number:
296                 means_by_iteration_number.append(self._geometric_mean(iteration_results))
297             final_mean = self._geometric_mean(group_means)
298             final_stdev = self._standard_deviation(means_by_iteration_number)
299             self.log('FINAL RESULTS\n')
300             self.log('Mean:\n-> {} ms\n'.format(final_mean))
301             self.log('Standard Deviation:\n-> {} ms ({}%)\n'.format(final_stdev, (final_stdev / final_mean) * 100))
302         except KeyboardInterrupt:
303             self._clean_up()
304             sys.exit(1)
305         finally:
306             self._clean_up()
307
308     def group_init(self):
309         pass
310
311     def run_iteration(self):
312         pass
313
314     def initialize(self):
315         pass
316
317     def will_parse_arguments(self):
318         pass
319
320     def did_parse_arguments(self, args):
321         pass