26f883d69fb891cbe3dc05133da033f610f6e083
[WebKit.git] / Tools / Scripts / webkitpy / benchmark_runner / benchmark_runner.py
1 #!/usr/bin/env python
2
3 import json
4 import logging
5 import shutil
6 import signal
7 import subprocess
8 import sys
9 import tempfile
10 import time
11 import types
12 import os
13 import urlparse
14
15 from benchmark_builder import BenchmarkBuilder
16 from benchmark_results import BenchmarkResults
17 from browser_driver.browser_driver_factory import BrowserDriverFactory
18 from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
19 from utils import timeout
20
21
22 _log = logging.getLogger(__name__)
23
24
25 class BenchmarkRunner(object):
26
27     def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, device_id=None):
28         try:
29             plan_file = self._find_plan_file(plan_file)
30             with open(plan_file, 'r') as fp:
31                 self._plan_name = os.path.split(os.path.splitext(plan_file)[0])[1]
32                 self._plan = json.load(fp)
33                 if local_copy:
34                     self._plan['local_copy'] = local_copy
35                 if count_override:
36                     self._plan['count'] = count_override
37                 self._browser_driver = BrowserDriverFactory.create(platform, browser)
38                 self._http_server_driver = HTTPServerDriverFactory.create(platform)
39                 self._http_server_driver.set_device_id(device_id)
40                 self._build_dir = os.path.abspath(build_dir) if build_dir else None
41                 self._output_file = output_file
42                 self._device_id = device_id
43         except IOError as error:
44             _log.error('Can not open plan file: {plan_file} - Error {error}'.format(plan_file=plan_file, error=error))
45             raise error
46         except ValueError as error:
47             _log.error('Plan file: {plan_file} may not follow JSON format - Error {error}'.format(plan_file=plan_file, error=error))
48             raise error
49
50     def _find_plan_file(self, plan_file):
51         if not os.path.exists(plan_file):
52             absPath = os.path.join(os.path.dirname(__file__), 'data/plans', plan_file)
53             if os.path.exists(absPath):
54                 return absPath
55             if not absPath.endswith('.plan'):
56                 absPath += '.plan'
57             if os.path.exists(absPath):
58                 return absPath
59         return plan_file
60
61     def _run_benchmark(self, count, web_root):
62         results = []
63         for iteration in xrange(1, count + 1):
64             _log.info('Start the iteration {current_iteration} of current benchmark'.format(current_iteration=iteration))
65             try:
66                 result = None
67                 self._http_server_driver.serve(web_root)
68                 self._browser_driver.prepare_env(self._device_id)
69                 url = urlparse.urljoin(self._http_server_driver.base_url(), self._plan_name + '/' + self._plan['entry_point'])
70                 self._browser_driver.launch_url(url, self._build_dir)
71                 with timeout(self._plan['timeout']):
72                     result = self._http_server_driver.fetch_result()
73                     assert(not self._http_server_driver.get_return_code())
74                     assert(result)
75                     results.append(json.loads(result))
76             finally:
77                 self._browser_driver.restore_env()
78                 self._browser_driver.close_browsers()
79                 self._http_server_driver.kill_server()
80             _log.info('End of {current_iteration} iteration of current benchmark'.format(current_iteration=iteration))
81         results = self._wrap(results)
82         self._dump(results, self._output_file if self._output_file else self._plan['output_file'])
83         self._show_results(results)
84
85     def execute(self):
86         with BenchmarkBuilder(self._plan_name, self._plan) as web_root:
87             self._run_benchmark(int(self._plan['count']), web_root)
88
89     @classmethod
90     def _dump(cls, results, output_file):
91         _log.info('Dumping the results to file')
92         try:
93             with open(output_file, 'w') as fp:
94                 json.dump(results, fp)
95         except IOError as error:
96             _log.error('Cannot open output file: {output_file} - Error: {error}'.format(output_file=output_file, error=error))
97             _log.error('Results are:\n {result}'.format(json.dumps(results)))
98
99     @classmethod
100     def _wrap(cls, dicts):
101         _log.debug('Merging following results:\n{results}'.format(results=json.dumps(dicts)))
102         if not dicts:
103             return None
104         ret = {}
105         for dic in dicts:
106             ret = cls._merge(ret, dic)
107         _log.debug('Results after merging:\n{result}'.format(result=json.dumps(ret)))
108         return ret
109
110     @classmethod
111     def _merge(cls, a, b):
112         assert(isinstance(a, type(b)))
113         arg_type = type(a)
114         # special handle for list type, and should be handle before equal check
115         if arg_type == types.ListType and len(a) and (type(a[0]) == types.StringType or type(a[0]) == types.UnicodeType):
116             return a
117         if arg_type == types.DictType:
118             result = {}
119             for key, value in a.items():
120                 if key in b:
121                     result[key] = cls._merge(value, b[key])
122                 else:
123                     result[key] = value
124             for key, value in b.items():
125                 if key not in result:
126                     result[key] = value
127             return result
128         # for other types
129         return a + b
130
131     @classmethod
132     def _show_results(cls, results):
133         results = BenchmarkResults(results)
134         print results.format()