Subclass Benchmark Runner script for WebDriver support
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Jul 2017 04:10:00 +0000 (04:10 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Jul 2017 04:10:00 +0000 (04:10 +0000)
https://bugs.webkit.org/show_bug.cgi?id=174390

Patch by Matthew Stewart <matthew_r_stewart@apple.com> on 2017-07-24
Reviewed by Dewei Zhu, Stephanie Lewis.

Add WebServerBenchmarkRunner and WebDriverBenchmarkRunner subclasses.

* Scripts/webkitpy/benchmark_runner/benchmark_builder.py:
(BenchmarkBuilder.__init__):
* Scripts/webkitpy/benchmark_runner/benchmark_runner.py:
(BenchmarkRunner.__init__):
(BenchmarkRunner._find_plan_file):
(BenchmarkRunner._run_one_test.and):
(BenchmarkRunner.execute):
(BenchmarkRunner._get_result): Deleted.
* Scripts/webkitpy/benchmark_runner/run_benchmark.py:
(parse_args):
(start):
* Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py: Added.
(WebDriverBenchmarkRunner):
(WebDriverBenchmarkRunner._get_result):
(WebDriverBenchmarkRunner._run_one_test):
* Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py: Added.
(WebServerBenchmarkRunner):
(WebServerBenchmarkRunner.__init__):
(WebServerBenchmarkRunner._get_result):
(WebServerBenchmarkRunner._run_one_test):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@219857 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/benchmark_runner/benchmark_builder.py
Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py
Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py
Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py [new file with mode: 0644]
Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py [new file with mode: 0644]

index 3919e63..8c98953 100644 (file)
@@ -1,5 +1,35 @@
 2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
 
+        Subclass Benchmark Runner script for WebDriver support
+        https://bugs.webkit.org/show_bug.cgi?id=174390
+
+        Reviewed by Dewei Zhu, Stephanie Lewis.
+
+        Add WebServerBenchmarkRunner and WebDriverBenchmarkRunner subclasses.
+
+        * Scripts/webkitpy/benchmark_runner/benchmark_builder.py:
+        (BenchmarkBuilder.__init__):
+        * Scripts/webkitpy/benchmark_runner/benchmark_runner.py:
+        (BenchmarkRunner.__init__):
+        (BenchmarkRunner._find_plan_file):
+        (BenchmarkRunner._run_one_test.and):
+        (BenchmarkRunner.execute):
+        (BenchmarkRunner._get_result): Deleted.
+        * Scripts/webkitpy/benchmark_runner/run_benchmark.py:
+        (parse_args):
+        (start):
+        * Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py: Added.
+        (WebDriverBenchmarkRunner):
+        (WebDriverBenchmarkRunner._get_result):
+        (WebDriverBenchmarkRunner._run_one_test):
+        * Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py: Added.
+        (WebServerBenchmarkRunner):
+        (WebServerBenchmarkRunner.__init__):
+        (WebServerBenchmarkRunner._get_result):
+        (WebServerBenchmarkRunner._run_one_test):
+
+2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
+
         Add BenchmarkRunner patches with WebDriver support for each testing plan
         https://bugs.webkit.org/show_bug.cgi?id=174443
 
index 6795a5b..71a006e 100644 (file)
@@ -8,17 +8,18 @@ import shutil
 import subprocess
 import tarfile
 
-from zipfile import ZipFile
 from webkitpy.benchmark_runner.utils import get_path_from_project_root, force_remove
+from zipfile import ZipFile
 
 
 _log = logging.getLogger(__name__)
 
 
 class BenchmarkBuilder(object):
-    def __init__(self, name, plan):
+    def __init__(self, name, plan, driver):
         self._name = name
         self._plan = plan
+        self._driver = driver
 
     def __enter__(self):
         self._web_root = tempfile.mkdtemp()
index 7ad2fbb..0081812 100644 (file)
@@ -15,14 +15,13 @@ import urlparse
 from benchmark_builder import BenchmarkBuilder
 from benchmark_results import BenchmarkResults
 from browser_driver.browser_driver_factory import BrowserDriverFactory
-from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
-from utils import timeout
 
 
 _log = logging.getLogger(__name__)
 
 
 class BenchmarkRunner(object):
+    name = 'benchmark_runner'
 
     def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, scale_unit=True, device_id=None):
         try:
@@ -37,8 +36,6 @@ class BenchmarkRunner(object):
                 if count_override:
                     self._plan['count'] = count_override
                 self._browser_driver = BrowserDriverFactory.create(platform, browser)
-                self._http_server_driver = HTTPServerDriverFactory.create(platform)
-                self._http_server_driver.set_device_id(device_id)
                 self._build_dir = os.path.abspath(build_dir) if build_dir else None
                 self._output_file = output_file
                 self._scale_unit = scale_unit
@@ -63,24 +60,8 @@ class BenchmarkRunner(object):
                 return absPath
         return plan_file
 
-    def _get_result(self, test_url):
-        result = self._browser_driver.add_additional_results(test_url, self._http_server_driver.fetch_result())
-        assert(not self._http_server_driver.get_return_code())
-        return result
-
     def _run_one_test(self, web_root, test_file):
-        result = None
-        try:
-            self._http_server_driver.serve(web_root)
-            url = urlparse.urljoin(self._http_server_driver.base_url(), self._plan_name + '/' + test_file)
-            self._browser_driver.launch_url(url, self._plan['options'], self._build_dir)
-            with timeout(self._plan['timeout']):
-                result = self._get_result(url)
-        finally:
-            self._browser_driver.close_browsers()
-            self._http_server_driver.kill_server()
-
-        return json.loads(result)
+        raise NotImplementedError('BenchmarkRunner is an abstract class and shouldn\'t be instantiated.')
 
     def _run_benchmark(self, count, web_root):
         results = []
@@ -118,7 +99,7 @@ class BenchmarkRunner(object):
         self.show_results(results, self._scale_unit)
 
     def execute(self):
-        with BenchmarkBuilder(self._plan_name, self._plan) as web_root:
+        with BenchmarkBuilder(self._plan_name, self._plan, self.name) as web_root:
             self._run_benchmark(int(self._plan['count']), web_root)
 
     @classmethod
index 7f6b808..bd6f415 100755 (executable)
@@ -7,11 +7,16 @@ import platform
 import os
 import sys
 
-from benchmark_runner import BenchmarkRunner
 from browser_driver.browser_driver_factory import BrowserDriverFactory
+from webdriver_benchmark_runner import WebDriverBenchmarkRunner
+from webserver_benchmark_runner import WebServerBenchmarkRunner
 
 
 _log = logging.getLogger(__name__)
+benchmark_runner_subclasses = {
+    WebDriverBenchmarkRunner.name: WebDriverBenchmarkRunner,
+    WebServerBenchmarkRunner.name: WebServerBenchmarkRunner,
+}
 
 
 def default_platform():
@@ -36,6 +41,7 @@ def parse_args():
     parser.add_argument('--local-copy', dest='localCopy', help='Path to a local copy of the benchmark. e.g. PerformanceTests/SunSpider/')
     parser.add_argument('--count', dest='countOverride', type=int, help='Number of times to run the benchmark. e.g. 5')
     parser.add_argument('--device-id', dest='device_id', default=None)
+    parser.add_argument('--driver', dest='driver', default=WebServerBenchmarkRunner.name, choices=benchmark_runner_subclasses.keys())
     parser.add_argument('--no-adjust-unit', dest='scale_unit', action='store_false')
     mutual_group = parser.add_mutually_exclusive_group(required=True)
     mutual_group.add_argument('--read-results-json', dest='json_file', help='Specify file you want to format')
@@ -88,7 +94,8 @@ def start(args):
         if failed:
             _log.error('The following benchmark plans have failed: %s' % failed)
         return len(failed)
-    runner = BenchmarkRunner(args.plan, args.localCopy, args.countOverride, args.buildDir, args.output, args.platform, args.browser, args.scale_unit, args.device_id)
+    benchmark_runner_class = benchmark_runner_subclasses[args.driver]
+    runner = benchmark_runner_class(args.plan, args.localCopy, args.countOverride, args.buildDir, args.output, args.platform, args.browser, args.scale_unit, args.device_id)
     runner.execute()
 
 
diff --git a/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py b/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py
new file mode 100644 (file)
index 0000000..1eb53f9
--- /dev/null
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+import json
+import logging
+
+from benchmark_runner import BenchmarkRunner
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+_log = logging.getLogger(__name__)
+
+
+class WebDriverBenchmarkRunner(BenchmarkRunner):
+    name = 'webdriver'
+
+    def _get_result(self, driver):
+        result = driver.execute_script("return window.webdriver_results")
+        return result
+
+    def _run_one_test(self, web_root, test_file):
+        result = None
+        try:
+            url = 'file://{root}/{plan_name}/{test_file}'.format(root=web_root, plan_name=self._plan_name, test_file=test_file)
+            driver = self._browser_driver.launch_driver(url, self._plan['options'], self._build_dir)
+            _log.info('Waiting on results from web browser')
+            result = WebDriverWait(driver, self._plan['timeout'], poll_frequency=1.0).until(self._get_result)
+        finally:
+            self._browser_driver.close_browsers()
+
+        return json.loads(result)
diff --git a/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py b/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py
new file mode 100644 (file)
index 0000000..9a97f6c
--- /dev/null
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import urlparse
+
+from benchmark_runner import BenchmarkRunner
+from http_server_driver.http_server_driver_factory import HTTPServerDriverFactory
+from utils import timeout
+
+
+_log = logging.getLogger(__name__)
+
+
+class WebServerBenchmarkRunner(BenchmarkRunner):
+    name = 'webserver'
+
+    def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, scale_unit=True, device_id=None):
+        self._http_server_driver = HTTPServerDriverFactory.create(platform)
+        self._http_server_driver.set_device_id(device_id)
+        super(WebServerBenchmarkRunner, self).__init__(plan_file, local_copy, count_override, build_dir, output_file, platform, browser, scale_unit, device_id)
+
+    def _get_result(self, test_url):
+        result = self._browser_driver.add_additional_results(test_url, self._http_server_driver.fetch_result())
+        assert(not self._http_server_driver.get_return_code())
+        return result
+
+    def _run_one_test(self, web_root, test_file):
+        result = None
+        try:
+            self._http_server_driver.serve(web_root)
+            url = urlparse.urljoin(self._http_server_driver.base_url(), self._plan_name + '/' + test_file)
+            self._browser_driver.launch_url(url, self._plan['options'], self._build_dir)
+            with timeout(self._plan['timeout']):
+                result = self._get_result(url)
+        finally:
+            self._browser_driver.close_browsers()
+            self._http_server_driver.kill_server()
+
+        return json.loads(result)