run-benchmark should print out the results
[WebKit-https.git] / Tools / Scripts / webkitpy / benchmark_runner / benchmark_runner.py
1 #!/usr/bin/env python
2
3 import json
4 import logging
5 import shutil
6 import signal
7 import subprocess
8 import tempfile
9 import time
10 import types
11 import os
12 import urlparse
13
14 from benchmark_builder.benchmark_builder_factory import BenchmarkBuilderFactory
15 from benchmark_results import BenchmarkResults
16 from browser_driver.browser_driver_factory import BrowserDriverFactory
17 from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
18 from utils import loadModule, getPathFromProjectRoot
19 from utils import timeout
20
21
22 _log = logging.getLogger(__name__)
23
24
25 class BenchmarkRunner(object):
26
27     def __init__(self, planFile, localCopy, countOverride, buildDir, outputFile, platform, browser):
28         _log.info('Initializing benchmark running')
29         try:
30             planFile = self._findPlanFile(planFile)
31             with open(planFile, 'r') as fp:
32                 self.planName = os.path.split(os.path.splitext(planFile)[0])[1]
33                 self.plan = json.load(fp)
34                 if localCopy:
35                     self.plan['local_copy'] = localCopy
36                 if countOverride:
37                     self.plan['count'] = countOverride
38                 self.browserDriver = BrowserDriverFactory.create([platform, browser])
39                 self.httpServerDriver = HTTPServerDriverFactory.create([self.plan['http_server_driver']])
40                 self.buildDir = os.path.abspath(buildDir) if buildDir else None
41                 self.outputFile = outputFile
42         except IOError:
43             _log.error('Can not open plan file: %s' % planFile)
44             raise
45         except ValueError:
46             _log.error('Plan file:%s may not follow JSON format' % planFile)
47             raise
48
49     def _findPlanFile(self, planFile):
50         if not os.path.exists(planFile):
51             absPath = os.path.join(os.path.dirname(__file__), 'data/plans', planFile)
52             if os.path.exists(absPath):
53                 return absPath
54             if not absPath.endswith('.plan'):
55                 absPath += '.plan'
56             if os.path.exists(absPath):
57                 return absPath
58         return planFile
59
60     def execute(self):
61         _log.info('Start to execute the plan')
62         _log.info('Start a new benchmark')
63         results = []
64         benchmarkBuilder = BenchmarkBuilderFactory.create([self.plan['benchmark_builder']])
65
66         if not self.plan.get('local_copy') and not self.plan.get('remote_archive'):
67             _log.error('Either local_copy or remote_archive must be specified in the plan')
68             return 2
69
70         webRoot = benchmarkBuilder.prepare(self.planName, self.plan.get('local_copy'), self.plan.get('remote_archive'),
71             self.plan.get('benchmark_patch'), self.plan.get('create_script'))
72         for x in xrange(int(self.plan['count'])):
73             _log.info('Start the iteration %d of current benchmark' % (x + 1))
74             self.httpServerDriver.serve(webRoot)
75             self.browserDriver.prepareEnv()
76             url = urlparse.urljoin(self.httpServerDriver.baseUrl(), self.planName + '/' + self.plan['entry_point'])
77             self.browserDriver.launchUrl(url, self.buildDir)
78             try:
79                 with timeout(self.plan['timeout']):
80                     result = json.loads(self.httpServerDriver.fetchResult())
81                 assert(not self.httpServerDriver.getReturnCode())
82                 assert(result)
83                 results.append(result)
84             except:
85                 _log.error('No result or server crashes. Something went wrong. Will skip current benchmark.')
86                 self.browserDriver.closeBrowsers()
87                 self.httpServerDriver.killServer()
88                 benchmarkBuilder.clean()
89                 return 1
90             finally:
91                 self.browserDriver.closeBrowsers()
92                 _log.info('End of %d iteration of current benchmark' % (x + 1))
93         results = self.wrap(results)
94         self.dump(results, self.outputFile if self.outputFile else self.plan['output_file'])
95         self.show_results(results)
96         benchmarkBuilder.clean()
97         return 0
98
99     @classmethod
100     def dump(cls, results, outputFile):
101         _log.info('Dumping the results to file')
102         try:
103             with open(outputFile, 'w') as fp:
104                 json.dump(results, fp)
105         except IOError:
106             _log.error('Cannot open output file: %s' % outputFile)
107             _log.error('Results are:\n %s', json.dumps(results))
108
109     @classmethod
110     def wrap(cls, dicts):
111         _log.debug('Merging following results:\n%s', json.dumps(dicts))
112         if not dicts:
113             return None
114         ret = {}
115         for dic in dicts:
116             ret = cls.merge(ret, dic)
117         _log.debug('Results after merging:\n%s', json.dumps(ret))
118         return ret
119
120     @classmethod
121     def merge(cls, a, b):
122         assert(isinstance(a, type(b)))
123         argType = type(a)
124         # special handle for list type, and should be handle before equal check
125         if argType == types.ListType and len(a) and (type(a[0]) == types.StringType or type(a[0]) == types.UnicodeType):
126             return a
127         if argType == types.DictType:
128             result = {}
129             for key, value in a.items():
130                 if key in b:
131                     result[key] = cls.merge(value, b[key])
132                 else:
133                     result[key] = value
134             for key, value in b.items():
135                 if key not in result:
136                     result[key] = value
137             return result
138         # for other types
139         return a + b
140
141     @classmethod
142     def show_results(cls, results):
143         results = BenchmarkResults(results)
144         print results.format()