Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import shutil
   6 import signal
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import time
  11 import types
  12 import os
  13 import urlparse
  14
  15 from benchmark_builder.benchmark_builder_factory import BenchmarkBuilderFactory
  16 from benchmark_results import BenchmarkResults
  17 from browser_driver.browser_driver_factory import BrowserDriverFactory
  18 from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
  19 from utils import loadModule, getPathFromProjectRoot
  20 from utils import timeout
  21
  22
  23 _log = logging.getLogger(__name__)
  24
  25
  26 class BenchmarkRunner(object):
  27
  28     def __init__(self, planFile, localCopy, countOverride, buildDir, outputFile, platform, browser, httpServerDriverOverride=None, deviceID=None):
  29         try:
  30             planFile = self._findPlanFile(planFile)
  31             with open(planFile, 'r') as fp:
  32                 self.planName = os.path.split(os.path.splitext(planFile)[0])[1]
  33                 self.plan = json.load(fp)
  34                 if localCopy:
  35                     self.plan['local_copy'] = localCopy
  36                 if countOverride:
  37                     self.plan['count'] = countOverride
  38                 if httpServerDriverOverride:
  39                     self.plan['http_server_driver'] = httpServerDriverOverride
  40                 self.browserDriver = BrowserDriverFactory.create([platform, browser])
  41                 self.httpServerDriver = HTTPServerDriverFactory.create([self.plan['http_server_driver']])
  42                 self.httpServerDriver.setDeviceID(deviceID)
  43                 self.buildDir = os.path.abspath(buildDir) if buildDir else None
  44                 self.outputFile = outputFile
  45                 self.deviceID = deviceID
  46         except IOError as error:
  47             _log.error('Can not open plan file: %s - Error %s' % (planFile, error))
  48             raise error
  49         except ValueError as error:
  50             _log.error('Plan file: %s may not follow JSON format - Error %s' % (planFile, error))
  51             raise error
  52
  53     def _findPlanFile(self, planFile):
  54         if not os.path.exists(planFile):
  55             absPath = os.path.join(os.path.dirname(__file__), 'data/plans', planFile)
  56             if os.path.exists(absPath):
  57                 return absPath
  58             if not absPath.endswith('.plan'):
  59                 absPath += '.plan'
  60             if os.path.exists(absPath):
  61                 return absPath
  62         return planFile
  63
  64     def execute(self):
  65         _log.info('Start to execute the plan')
  66         _log.info('Start a new benchmark')
  67         results = []
  68         self.benchmarkBuilder = BenchmarkBuilderFactory.create([self.plan['benchmark_builder']])
  69
  70         webRoot = self.benchmarkBuilder.prepare(self.planName, self.plan)
  71         for x in xrange(int(self.plan['count'])):
  72             _log.info('Start the iteration %d of current benchmark' % (x + 1))
  73             self.httpServerDriver.serve(webRoot)
  74             self.browserDriver.prepareEnv(self.deviceID)
  75             url = urlparse.urljoin(self.httpServerDriver.baseUrl(), self.planName + '/' + self.plan['entry_point'])
  76             self.browserDriver.launchUrl(url, self.buildDir)
  77             result = None
  78             try:
  79                 with timeout(self.plan['timeout']):
  80                     result = self.httpServerDriver.fetchResult()
  81                 assert(not self.httpServerDriver.getReturnCode())
  82                 assert(result)
  83                 results.append(json.loads(result))
  84             except Exception as error:
  85                 _log.error('No result or the server crashed. Something went wrong. Will skip current benchmark.\nError: %s, Server return code: %d, result: %s' % (error, not self.httpServerDriver.getReturnCode(), result))
  86                 self.cleanup()
  87                 sys.exit(1)
  88             finally:
  89                 self.browserDriver.closeBrowsers()
  90                 _log.info('End of %d iteration of current benchmark' % (x + 1))
  91         results = self.wrap(results)
  92         self.dump(results, self.outputFile if self.outputFile else self.plan['output_file'])
  93         self.show_results(results)
  94         self.benchmarkBuilder.clean()
  95         sys.exit()
  96
  97     def cleanup(self):
  98         if self.browserDriver:
  99             self.browserDriver.closeBrowsers()
 100         if self.httpServerDriver:
 101             self.httpServerDriver.killServer()
 102         if self.benchmarkBuilder:
 103             self.benchmarkBuilder.clean()
 104
 105     @classmethod
 106     def dump(cls, results, outputFile):
 107         _log.info('Dumping the results to file')
 108         try:
 109             with open(outputFile, 'w') as fp:
 110                 json.dump(results, fp)
 111         except IOError as error:
 112             _log.error('Cannot open output file: %s - Error: %s' % (outputFile, error))
 113             _log.error('Results are:\n %s', json.dumps(results))
 114
 115     @classmethod
 116     def wrap(cls, dicts):
 117         _log.debug('Merging following results:\n%s', json.dumps(dicts))
 118         if not dicts:
 119             return None
 120         ret = {}
 121         for dic in dicts:
 122             ret = cls.merge(ret, dic)
 123         _log.debug('Results after merging:\n%s', json.dumps(ret))
 124         return ret
 125
 126     @classmethod
 127     def merge(cls, a, b):
 128         assert(isinstance(a, type(b)))
 129         argType = type(a)
 130         # special handle for list type, and should be handle before equal check
 131         if argType == types.ListType and len(a) and (type(a[0]) == types.StringType or type(a[0]) == types.UnicodeType):
 132             return a
 133         if argType == types.DictType:
 134             result = {}
 135             for key, value in a.items():
 136                 if key in b:
 137                     result[key] = cls.merge(value, b[key])
 138                 else:
 139                     result[key] = value
 140             for key, value in b.items():
 141                 if key not in result:
 142                     result[key] = value
 143             return result
 144         # for other types
 145         return a + b
 146
 147     @classmethod
 148     def show_results(cls, results):
 149         results = BenchmarkResults(results)
 150         print results.format()