47e78bd5a222a70dcca00259ba49f22b618df81e
[WebKit-https.git] / Tools / Scripts / webkitpy / benchmark_runner / benchmark_runner.py
1 #!/usr/bin/env python
2
3 import json
4 import logging
5 import shutil
6 import signal
7 import subprocess
8 import sys
9 import tempfile
10 import time
11 import types
12 import os
13 import urlparse
14
15 from benchmark_builder.benchmark_builder_factory import BenchmarkBuilderFactory
16 from benchmark_results import BenchmarkResults
17 from browser_driver.browser_driver_factory import BrowserDriverFactory
18 from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
19 from utils import loadModule, getPathFromProjectRoot
20 from utils import timeout
21
22
23 _log = logging.getLogger(__name__)
24
25
26 class BenchmarkRunner(object):
27
28     def __init__(self, planFile, localCopy, countOverride, buildDir, outputFile, platform, browser, httpServerDriverOverride=None, deviceID=None):
29         try:
30             planFile = self._findPlanFile(planFile)
31             with open(planFile, 'r') as fp:
32                 self.planName = os.path.split(os.path.splitext(planFile)[0])[1]
33                 self.plan = json.load(fp)
34                 if localCopy:
35                     self.plan['local_copy'] = localCopy
36                 if countOverride:
37                     self.plan['count'] = countOverride
38                 if httpServerDriverOverride:
39                     self.plan['http_server_driver'] = httpServerDriverOverride
40                 self.browserDriver = BrowserDriverFactory.create([platform, browser])
41                 self.httpServerDriver = HTTPServerDriverFactory.create([self.plan['http_server_driver']])
42                 self.httpServerDriver.setDeviceID(deviceID)
43                 self.buildDir = os.path.abspath(buildDir) if buildDir else None
44                 self.outputFile = outputFile
45                 self.deviceID = deviceID
46         except IOError as error:
47             _log.error('Can not open plan file: %s - Error %s' % (planFile, error))
48             raise error
49         except ValueError as error:
50             _log.error('Plan file: %s may not follow JSON format - Error %s' % (planFile, error))
51             raise error
52
53     def _findPlanFile(self, planFile):
54         if not os.path.exists(planFile):
55             absPath = os.path.join(os.path.dirname(__file__), 'data/plans', planFile)
56             if os.path.exists(absPath):
57                 return absPath
58             if not absPath.endswith('.plan'):
59                 absPath += '.plan'
60             if os.path.exists(absPath):
61                 return absPath
62         return planFile
63
64     def execute(self):
65         _log.info('Start to execute the plan')
66         _log.info('Start a new benchmark')
67         results = []
68         self.benchmarkBuilder = BenchmarkBuilderFactory.create([self.plan['benchmark_builder']])
69
70         webRoot = self.benchmarkBuilder.prepare(self.planName, self.plan)
71         for x in xrange(int(self.plan['count'])):
72             _log.info('Start the iteration %d of current benchmark' % (x + 1))
73             self.httpServerDriver.serve(webRoot)
74             self.browserDriver.prepareEnv(self.deviceID)
75             url = urlparse.urljoin(self.httpServerDriver.baseUrl(), self.planName + '/' + self.plan['entry_point'])
76             self.browserDriver.launchUrl(url, self.buildDir)
77             result = None
78             try:
79                 with timeout(self.plan['timeout']):
80                     result = self.httpServerDriver.fetchResult()
81                 assert(not self.httpServerDriver.getReturnCode())
82                 assert(result)
83                 results.append(json.loads(result))
84             except Exception as error:
85                 _log.error('No result or the server crashed. Something went wrong. Will skip current benchmark.\nError: %s, Server return code: %d, result: %s' % (error, not self.httpServerDriver.getReturnCode(), result))
86                 self.cleanup()
87                 sys.exit(1)
88             finally:
89                 self.browserDriver.closeBrowsers()
90                 _log.info('End of %d iteration of current benchmark' % (x + 1))
91         results = self.wrap(results)
92         self.dump(results, self.outputFile if self.outputFile else self.plan['output_file'])
93         self.show_results(results)
94         self.benchmarkBuilder.clean()
95         sys.exit()
96
97     def cleanup(self):
98         if self.browserDriver:
99             self.browserDriver.closeBrowsers()
100         if self.httpServerDriver:
101             self.httpServerDriver.killServer()
102         if self.benchmarkBuilder:
103             self.benchmarkBuilder.clean()
104
105     @classmethod
106     def dump(cls, results, outputFile):
107         _log.info('Dumping the results to file')
108         try:
109             with open(outputFile, 'w') as fp:
110                 json.dump(results, fp)
111         except IOError as error:
112             _log.error('Cannot open output file: %s - Error: %s' % (outputFile, error))
113             _log.error('Results are:\n %s', json.dumps(results))
114
115     @classmethod
116     def wrap(cls, dicts):
117         _log.debug('Merging following results:\n%s', json.dumps(dicts))
118         if not dicts:
119             return None
120         ret = {}
121         for dic in dicts:
122             ret = cls.merge(ret, dic)
123         _log.debug('Results after merging:\n%s', json.dumps(ret))
124         return ret
125
126     @classmethod
127     def merge(cls, a, b):
128         assert(isinstance(a, type(b)))
129         argType = type(a)
130         # special handle for list type, and should be handle before equal check
131         if argType == types.ListType and len(a) and (type(a[0]) == types.StringType or type(a[0]) == types.UnicodeType):
132             return a
133         if argType == types.DictType:
134             result = {}
135             for key, value in a.items():
136                 if key in b:
137                     result[key] = cls.merge(value, b[key])
138                 else:
139                     result[key] = value
140             for key, value in b.items():
141                 if key not in result:
142                     result[key] = value
143             return result
144         # for other types
145         return a + b
146
147     @classmethod
148     def show_results(cls, results):
149         results = BenchmarkResults(results)
150         print results.format()