Make arguments of run-benchmark more user friendly
[WebKit-https.git] / Tools / Scripts / webkitpy / benchmark_runner / benchmark_runner.py
index b4b9d19af0ab012beac0267d555dfccf239cdc47..179287d7a20b38304deddc10016cd848ccd641a0 100644 (file)
@@ -26,51 +26,65 @@ class BenchmarkRunner(object):
     def __init__(self, planFile, buildDir, outputFile, platform, browser):
         _log.info('Initializing benchmark running')
         try:
+            planFile = self._findPlanFile(planFile)
             with open(planFile, 'r') as fp:
                 self.plan = json.load(fp)
                 self.browserDriver = BrowserDriverFactory.create([platform, browser])
                 self.httpServerDriver = HTTPServerDriverFactory.create([self.plan['http_server_driver']])
-                self.benchmarks = self.plan['benchmarks']
-                self.buildDir = os.path.abspath(buildDir)
+                self.buildDir = os.path.abspath(buildDir) if buildDir else None
                 self.outputFile = outputFile
         except IOError:
             _log.error('Can not open plan file: %s' % planFile)
+            raise
         except ValueError:
-            _log.error('Plan file:%s may not follow json format' % planFile)
-        except:
+            _log.error('Plan file:%s may not follow JSON format' % planFile)
             raise
 
+    def _findPlanFile(self, planFile):
+        if not os.path.exists(planFile):
+            absPath = os.path.join(os.path.dirname(__file__), 'data/plans', planFile)
+            if os.path.exists(absPath):
+                return absPath
+            if not absPath.endswith('.plan'):
+                absPath += '.plan'
+            if os.path.exists(absPath):
+                return absPath
+        return planFile
+
     def execute(self):
         _log.info('Start to execute the plan')
-        for benchmark in self.benchmarks:
-            _log.info('Start a new benchmark')
-            results = []
-            benchmarkBuilder = BenchmarkBuilderFactory.create([benchmark['benchmark_builder']])
-            webRoot = benchmarkBuilder.prepare(benchmark['original_benchmark'], benchmark['benchmark_patch'] if 'benchmark_patch' in benchmark else None)
-            for x in xrange(int(benchmark['count'])):
-                _log.info('Start the iteration %d of current benchmark' % (x + 1))
-                self.httpServerDriver.serve(webRoot)
-                self.browserDriver.prepareEnv()
-                self.browserDriver.launchUrl(urlparse.urljoin(self.httpServerDriver.baseUrl(), benchmark['entry_point']), self.buildDir)
-                try:
-                    with timeout(benchmark['timeout']):
-                        result = json.loads(self.httpServerDriver.fetchResult())
-                        assert(result)
-                        results.append(result)
-                except:
-                    _log.error('No result. Something went wrong. Will skip current benchmark.')
-                    self.browserDriver.closeBrowsers()
-                    break
-                finally:
-                    self.browserDriver.closeBrowsers()
-                    _log.info('End of %d iteration of current benchmark' % (x + 1))
-            results = self.wrap(results)
-            self.dump(results, self.outputFile if self.outputFile else benchmark['output_file'])
-            benchmarkBuilder.clean()
+        _log.info('Start a new benchmark')
+        results = []
+        benchmarkBuilder = BenchmarkBuilderFactory.create([self.plan['benchmark_builder']])
+        webRoot = benchmarkBuilder.prepare(self.plan['original_benchmark'], self.plan['benchmark_patch'] if 'benchmark_patch' in self.plan else None)
+        for x in xrange(int(self.plan['count'])):
+            _log.info('Start the iteration %d of current benchmark' % (x + 1))
+            self.httpServerDriver.serve(webRoot)
+            self.browserDriver.prepareEnv()
+            self.browserDriver.launchUrl(urlparse.urljoin(self.httpServerDriver.baseUrl(), self.plan['entry_point']), self.buildDir)
+            try:
+                with timeout(self.plan['timeout']):
+                    result = json.loads(self.httpServerDriver.fetchResult())
+                assert(not self.httpServerDriver.getReturnCode())
+                assert(result)
+                results.append(result)
+            except:
+                _log.error('No result or server crashes. Something went wrong. Will skip current benchmark.')
+                self.browserDriver.closeBrowsers()
+                self.httpServerDriver.killServer()
+                benchmarkBuilder.clean()
+                return 1
+            finally:
+                self.browserDriver.closeBrowsers()
+                _log.info('End of %d iteration of current benchmark' % (x + 1))
+        results = self.wrap(results)
+        self.dump(results, self.outputFile if self.outputFile else self.plan['output_file'])
+        benchmarkBuilder.clean()
+        return 0
 
     @classmethod
     def dump(cls, results, outputFile):
-        _log.info('Dumpping the results to file')
+        _log.info('Dumping the results to file')
         try:
             with open(outputFile, 'w') as fp:
                 json.dump(results, fp)
@@ -80,11 +94,13 @@ class BenchmarkRunner(object):
 
     @classmethod
     def wrap(cls, dicts):
+        _log.info('Merging following results:\n%s', json.dumps(dicts))
         if not dicts:
             return None
         ret = {}
         for dic in dicts:
             ret = cls.merge(ret, dic)
+        _log.info('Results after merging:\n%s', json.dumps(ret))
         return ret
 
     @classmethod
@@ -92,9 +108,7 @@ class BenchmarkRunner(object):
         assert(isinstance(a, type(b)))
         argType = type(a)
         # special handle for list type, and should be handle before equal check
-        if argType == types.ListType:
-            return a + b
-        if a == b:
+        if argType == types.ListType and len(a) and (type(a[0]) == types.StringType or type(a[0]) == types.UnicodeType):
             return a
         if argType == types.DictType:
             result = {}