Add "-o/--output" option to startup.py and new_tab.py benchmark scripts to save the...
[WebKit-https.git] / PerformanceTests / LaunchTime / launch_time.py
1 import SimpleHTTPServer
2 import SocketServer
3 import argparse
4 import logging
5 from math import sqrt
6 from operator import mul
7 import os
8 import json
9 from subprocess import call, check_output
10 import sys
11 import threading
12 import time
13
14
15
16 # Supress logs from feedback server
17 logging.getLogger().setLevel(logging.FATAL)
18
19
20 class DefaultLaunchTimeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
21     def get_test_page(self):
22         return '''<!DOCTYPE html>
23         <html>
24           <head>
25             <title>Launch Time Benchmark</title>
26             <meta http-equiv="Content-Type" content="text/html" />
27             <script>
28                 function sendDone() {
29                     const time = performance.timing.navigationStart
30                     const request = new XMLHttpRequest();
31                     request.open("POST", "done", false);
32                     request.setRequestHeader('Content-Type', 'application/json');
33                     request.send(JSON.stringify(time));
34                 }
35                 window.onload = sendDone;
36             </script>
37           </head>
38           <body>
39             <h1>New Tab Benchmark</h1>
40           </body>
41         </html>
42         '''
43
44     def on_receive_stop_signal(self, data):
45         pass
46
47     def do_HEAD(self):
48         self.send_response(200)
49         self.send_header('Content-type', 'text/html')
50         self.end_headers()
51
52     def do_GET(self):
53         self.send_response(200)
54         self.send_header('Content-type', 'text/html')
55         self.end_headers()
56         if not self.path.startswith('/blank'):
57             self.wfile.write(self.get_test_page())
58         self.wfile.close()
59
60     def do_POST(self):
61         self.send_response(200)
62         self.send_header('Content-type', 'text/html')
63         self.end_headers()
64         self.wfile.write('done')
65         self.wfile.close()
66
67         data_string = self.rfile.read(int(self.headers['Content-Length']))
68         self.on_receive_stop_signal(data_string)
69
70     def log_message(self, format, *args):
71         pass
72
73
74 class LaunchTimeBenchmark:
75     def __init__(self):
76         self._server_ready = threading.Semaphore(0)
77         self._server = None
78         self._server_thread = None
79         self._port = 8080
80         self._feedback_port = None
81         self._feedback_server = None
82         self._open_count = 0
83         self._app_name = None
84         self._verbose = False
85         self._feedback_in_browser = False
86         self._save_results_to_json = False
87         self._json_results_path = None
88         self._do_not_ignore_first_result = False
89         self._iterations = 5
90         self._browser_bundle_path = '/Applications/Safari.app'
91         self.response_handler = None
92         self.benchmark_description = None
93         self.use_geometric_mean = False
94         self.wait_time_high = 1
95         self.wait_time_low = 0.1
96         self.iteration_groups = 1
97         self.initialize()
98
99     def _parse_browser_bundle_path(self, path):
100         if not os.path.isdir(path) or not path.endswith('.app'):
101             raise argparse.ArgumentTypeError(
102                 'Invalid app bundle path: "{}"'.format(path))
103         return path
104
105     def _parse_args(self):
106         self.argument_parser = argparse.ArgumentParser(description=self.benchmark_description)
107         self.argument_parser.add_argument('-p', '--path', type=self._parse_browser_bundle_path,
108             help='path for browser application bundle (default: {})'.format(self._browser_bundle_path))
109         self.argument_parser.add_argument('-n', '--iterations', type=int,
110             help='number of iterations of test (default: {})'.format(self._iterations))
111         self.argument_parser.add_argument('-v', '--verbose', action='store_true',
112             help="print each iteration's time")
113         self.argument_parser.add_argument('-f', '--feedback-in-browser', action='store_true',
114             help="show benchmark results in browser (default: {})".format(self._feedback_in_browser))
115         self.argument_parser.add_argument('-o', '--output', type=self._json_results_path,
116             help='saves benchmark results in json format (default: {})'.format(self._json_results_path))
117         self.will_parse_arguments()
118
119         args = self.argument_parser.parse_args()
120         if args.iterations:
121             self._iterations = args.iterations
122         if args.path:
123             self._browser_bundle_path = args.path
124         if args.verbose is not None:
125             self._verbose = args.verbose
126         if args.feedback_in_browser is not None:
127             self._feedback_in_browser = args.feedback_in_browser
128         if args.output:
129             self._save_results_to_json = True
130             self._json_results_path = args.output
131         path_len = len(self._browser_bundle_path)
132         start_index = self._browser_bundle_path.rfind('/', 0, path_len)
133         end_index = self._browser_bundle_path.rfind('.', 0, path_len)
134         self._app_name = self._browser_bundle_path[start_index + 1:end_index]
135         self.did_parse_arguments(args)
136
137     def _run_server(self):
138         self._server_ready.release()
139         self._server.serve_forever()
140
141     def _setup_servers(self):
142         while True:
143             try:
144                 self._server = SocketServer.TCPServer(
145                     ('0.0.0.0', self._port), self.response_handler)
146                 break
147             except:
148                 self._port += 1
149         print 'Running test server at http://localhost:{}'.format(self._port)
150
151         self._server_thread = threading.Thread(target=self._run_server)
152         self._server_thread.start()
153         self._server_ready.acquire()
154
155         if self._feedback_in_browser:
156             from feedback_server import FeedbackServer
157             self._feedback_server = FeedbackServer()
158             self._feedback_port = self._feedback_server.start()
159
160     def _clean_up(self):
161         self._server.shutdown()
162         self._server_thread.join()
163         if self._feedback_in_browser:
164             self._feedback_server.stop()
165
166     def _exit_due_to_exception(self, reason):
167         self.log(reason)
168         self._clean_up()
169         sys.exit(1)
170
171     def _geometric_mean(self, values):
172         product = reduce(mul, values)
173         return product ** (1.0 / len(values))
174
175     def _standard_deviation(self, results, mean=None):
176         if mean is None:
177             mean = sum(results) / float(len(results))
178         divisor = float(len(results) - 1) if len(results) > 1 else float(len(results))
179         variance = sum((x - mean) ** 2 for x in results) / divisor
180         return sqrt(variance)
181
182     def _compute_results(self, results):
183         if not results:
184             self._exit_due_to_exception('No results to compute.\n')
185         if len(results) > 1 and not self._do_not_ignore_first_result:
186             results = results[1:]
187         mean = sum(results) / float(len(results))
188         stdev = self._standard_deviation(results, mean)
189         return mean, stdev
190
191     def _wait_times(self):
192         if self.iteration_groups == 1:
193             yield self.wait_time_high
194             return
195         increment_per_group = float(self.wait_time_high - self.wait_time_low) / (self.iteration_groups - 1)
196         for i in range(self.iteration_groups):
197             yield self.wait_time_low + increment_per_group * i
198
199     def open_tab(self, blank=False):
200         if blank:
201             call(['open', '-a', self._browser_bundle_path,
202                 'http://localhost:{}/blank/{}'.format(self._port, self._open_count)])
203         else:
204             call(['open', '-a', self._browser_bundle_path,
205                 'http://localhost:{}/{}'.format(self._port, self._open_count)])
206         self._open_count += 1
207
208     def launch_browser(self):
209         if self._feedback_in_browser:
210             call(['open', '-a', self._browser_bundle_path,
211                 'http://localhost:{}'.format(self._feedback_port), '-F'])
212             self._feedback_server.wait_until_client_has_loaded()
213         else:
214             call(['open', '-a', self._browser_bundle_path,
215                 'http://localhost:{}/blank'.format(self._port), '-F'])
216         self.wait(2)
217
218     def quit_browser(self):
219         def quit_app():
220             call(['osascript', '-e', 'quit app "{}"'.format(self._browser_bundle_path)])
221
222         def is_app_closed():
223             out = check_output(['osascript', '-e', 'tell application "System Events"',
224                 '-e', 'copy (get name of every process whose name is "{}") to stdout'.format(self._app_name),
225                 '-e', 'end tell'])
226             return len(out.strip()) == 0
227
228         while not is_app_closed():
229             quit_app()
230         self.wait(1)
231
232     def close_tab(self):
233         call(['osascript', '-e',
234             'tell application "System Events" to keystroke "w" using command down'])
235
236     def wait(self, duration):
237         wait_start = time.time()
238         while time.time() - wait_start < duration:
239             pass
240
241     def log(self, message):
242         if self._feedback_in_browser:
243             self._feedback_server.send_message(message)
244         sys.stdout.write(message)
245         sys.stdout.flush()
246
247     def log_verbose(self, message):
248         if self._verbose:
249             self.log(message)
250
251     def run(self):
252         self._parse_args()
253         self._setup_servers()
254         self.quit_browser()
255         print ''
256
257         try:
258             group_means = []
259             if self._save_results_to_json:
260                 resultsDict = {self.get_test_name(): {"metrics": {"Time": {"current": []}}}}
261
262             results_by_iteration_number = [[] for _ in range(self._iterations)]
263
264             group = 1
265             for wait_duration in self._wait_times():
266                 self.group_init()
267                 if self.iteration_groups > 1:
268                     self.log('Running group {}{}'.format(group, ':\n' if self._verbose else '...'))
269
270                 results = []
271                 for i in range(self._iterations):
272                     try:
273                         if not self._verbose:
274                             self.log('.')
275                         result_in_ms = self.run_iteration()
276                         self.log_verbose('({}) {} ms\n'.format(i + 1, result_in_ms))
277                         self.wait(wait_duration)
278                         results.append(result_in_ms)
279                         results_by_iteration_number[i].append(result_in_ms)
280                     except KeyboardInterrupt:
281                         raise KeyboardInterrupt
282                     except Exception as error:
283                         self._exit_due_to_exception('(Test {} failed) {}: {}\n'.format(i + 1 if self._verbose else i, type(error).__name__, error))
284                 if not self._verbose:
285                     print ''
286
287                 if self._save_results_to_json:
288                     resultsDict[self.get_test_name()]["metrics"]["Time"]["current"].append(results)
289
290                 mean, stdev = self._compute_results(results)
291                 self.log_verbose('RESULTS:\n')
292                 self.log_verbose('mean: {} ms\n'.format(mean))
293                 self.log_verbose('std dev: {} ms ({}%)\n\n'.format(stdev, (stdev / mean) * 100))
294                 if self._verbose:
295                     self.wait(1)
296                 group_means.append(mean)
297                 group += 1
298                 self.quit_browser()
299
300             if not self._verbose:
301                 print '\n'
302
303             if self._feedback_in_browser:
304                 self.launch_browser()
305
306             if self._save_results_to_json and self._json_results_path:
307                 with open(self._json_results_path, "w") as jsonFile:
308                     json.dump(resultsDict, jsonFile, indent=4, separators=(',', ': '))
309
310             means_by_iteration_number = []
311             if len(results_by_iteration_number) > 1 and not self._do_not_ignore_first_result:
312                 results_by_iteration_number = results_by_iteration_number[1:]
313             for iteration_results in results_by_iteration_number:
314                 means_by_iteration_number.append(self._geometric_mean(iteration_results))
315             final_mean = self._geometric_mean(group_means)
316             final_stdev = self._standard_deviation(means_by_iteration_number)
317             self.log('FINAL RESULTS\n')
318             self.log('Mean:\n-> {} ms\n'.format(final_mean))
319             self.log('Standard Deviation:\n-> {} ms ({}%)\n'.format(final_stdev, (final_stdev / final_mean) * 100))
320         except KeyboardInterrupt:
321             self._clean_up()
322             sys.exit(1)
323         finally:
324             self._clean_up()
325
326     def group_init(self):
327         pass
328
329     def run_iteration(self):
330         pass
331
332     def initialize(self):
333         pass
334
335     def will_parse_arguments(self):
336         pass
337
338     def did_parse_arguments(self, args):
339         pass
340
341     def get_test_name(self):
342         return "LaunchTimeBenchmark"