Increase stablility of run-benchmark script
[WebKit-https.git] / Tools / Scripts / webkitpy / benchmark_runner / benchmark_runner.py
1 #!/usr/bin/env python
2
3 import json
4 import logging
5 import shutil
6 import signal
7 import subprocess
8 import tempfile
9 import time
10 import types
11 import os
12 import urlparse
13
14 from benchmark_builder.benchmark_builder_factory import BenchmarkBuilderFactory
15 from browser_driver.browser_driver_factory import BrowserDriverFactory
16 from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
17 from utils import loadModule, getPathFromProjectRoot
18 from utils import timeout
19
20
21 _log = logging.getLogger(__name__)
22
23
24 class BenchmarkRunner(object):
25
26     def __init__(self, planFile, buildDir, outputFile, platform, browser):
27         _log.info('Initializing benchmark running')
28         try:
29             with open(planFile, 'r') as fp:
30                 self.plan = json.load(fp)
31                 self.browserDriver = BrowserDriverFactory.create([platform, browser])
32                 self.httpServerDriver = HTTPServerDriverFactory.create([self.plan['http_server_driver']])
33                 self.buildDir = os.path.abspath(buildDir)
34                 self.outputFile = outputFile
35         except IOError:
36             _log.error('Can not open plan file: %s' % planFile)
37         except ValueError:
38             _log.error('Plan file:%s may not follow JSON format' % planFile)
39         except:
40             raise
41
42     def execute(self):
43         _log.info('Start to execute the plan')
44         _log.info('Start a new benchmark')
45         results = []
46         benchmarkBuilder = BenchmarkBuilderFactory.create([self.plan['benchmark_builder']])
47         webRoot = benchmarkBuilder.prepare(self.plan['original_benchmark'], self.plan['benchmark_patch'] if 'benchmark_patch' in self.plan else None)
48         for x in xrange(int(self.plan['count'])):
49             _log.info('Start the iteration %d of current benchmark' % (x + 1))
50             self.httpServerDriver.serve(webRoot)
51             self.browserDriver.prepareEnv()
52             self.browserDriver.launchUrl(urlparse.urljoin(self.httpServerDriver.baseUrl(), self.plan['entry_point']), self.buildDir)
53             try:
54                 with timeout(self.plan['timeout']):
55                     result = json.loads(self.httpServerDriver.fetchResult())
56                 assert(not self.httpServerDriver.getReturnCode())
57                 assert(result)
58                 results.append(result)
59             except:
60                 _log.error('No result or server crashes. Something went wrong. Will skip current benchmark.')
61                 self.browserDriver.closeBrowsers()
62                 self.httpServerDriver.killServer()
63                 benchmarkBuilder.clean()
64                 return 1
65             finally:
66                 self.browserDriver.closeBrowsers()
67                 _log.info('End of %d iteration of current benchmark' % (x + 1))
68         results = self.wrap(results)
69         self.dump(results, self.outputFile if self.outputFile else self.plan['output_file'])
70         benchmarkBuilder.clean()
71         return 0
72
73     @classmethod
74     def dump(cls, results, outputFile):
75         _log.info('Dumpping the results to file')
76         try:
77             with open(outputFile, 'w') as fp:
78                 json.dump(results, fp)
79         except IOError:
80             _log.error('Cannot open output file: %s' % outputFile)
81             _log.error('Results are:\n %s', json.dumps(results))
82
83     @classmethod
84     def wrap(cls, dicts):
85         if not dicts:
86             return None
87         ret = {}
88         for dic in dicts:
89             ret = cls.merge(ret, dic)
90         return ret
91
92     @classmethod
93     def merge(cls, a, b):
94         assert(isinstance(a, type(b)))
95         argType = type(a)
96         # special handle for list type, and should be handle before equal check
97         if argType == types.ListType:
98             return a + b
99         if a == b:
100             return a
101         if argType == types.DictType:
102             result = {}
103             for key, value in a.items():
104                 if key in b:
105                     result[key] = cls.merge(value, b[key])
106                 else:
107                     result[key] = value
108             for key, value in b.items():
109                 if key not in result:
110                     result[key] = value
111             return result
112         # for other types
113         return a + b