Add functions to autoinstaller needed for Benchmark Runner script
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Jul 2017 01:42:03 +0000 (01:42 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Jul 2017 01:42:03 +0000 (01:42 +0000)
https://bugs.webkit.org/show_bug.cgi?id=174331

Patch by Matthew Stewart <matthew_r_stewart@apple.com> on 2017-07-24
Reviewed by Dean Johnson, Dewei Zhu, Stephanie Lewis.

Adds autoinstaller functions to install selenium and webdriver binaries
which will be used by the browser driver part of Benchmark Runner.

* Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:
(BrowserDriver.restore_env):
(BrowserDriver):
(BrowserDriver.get_webdriver_binary_path):
* Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py:
(LinuxChromeDriver.launch_driver):
* Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py:
(LinuxFirefoxDriver.launch_driver):
* Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py:
(OSXChromeDriver.launch_driver):
(OSXChromeCanaryDriver.launch_driver):
* Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py:
(OSXFirefoxDriver.launch_driver):
(OSXFirefoxNightlyDriver.launch_driver):
* Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py:
* Scripts/webkitpy/benchmark_runner/utils.py:
(get_driver_binary_path):
* Scripts/webkitpy/common/system/autoinstall.py:
(AutoInstaller._unzip):
* Scripts/webkitpy/thirdparty/__init__.py:
(AutoinstallImportHook.find_module):
(AutoinstallImportHook._install_selenium):
(AutoinstallImportHook):
(AutoinstallImportHook._install_chromedriver):
(AutoinstallImportHook._install_geckodriver):
(AutoinstallImportHook.get_latest_pypi_url):
(AutoinstallImportHook.install_binary):
(autoinstall_everything):
(get_driver_filename):
(get_os_info):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@219853 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py
Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py
Tools/Scripts/webkitpy/benchmark_runner/utils.py
Tools/Scripts/webkitpy/common/system/autoinstall.py
Tools/Scripts/webkitpy/thirdparty/__init__.py

index cf2277e..4f13266 100644 (file)
@@ -1,5 +1,46 @@
 2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
 
+        Add functions to autoinstaller needed for Benchmark Runner script
+        https://bugs.webkit.org/show_bug.cgi?id=174331
+
+        Reviewed by Dean Johnson, Dewei Zhu, Stephanie Lewis.
+
+        Adds autoinstaller functions to install selenium and webdriver binaries
+        which will be used by the browser driver part of Benchmark Runner.
+
+        * Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:
+        (BrowserDriver.restore_env):
+        (BrowserDriver):
+        (BrowserDriver.get_webdriver_binary_path):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py:
+        (LinuxChromeDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py:
+        (LinuxFirefoxDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py:
+        (OSXChromeDriver.launch_driver):
+        (OSXChromeCanaryDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py:
+        (OSXFirefoxDriver.launch_driver):
+        (OSXFirefoxNightlyDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py:
+        * Scripts/webkitpy/benchmark_runner/utils.py:
+        (get_driver_binary_path):
+        * Scripts/webkitpy/common/system/autoinstall.py:
+        (AutoInstaller._unzip):
+        * Scripts/webkitpy/thirdparty/__init__.py:
+        (AutoinstallImportHook.find_module):
+        (AutoinstallImportHook._install_selenium):
+        (AutoinstallImportHook):
+        (AutoinstallImportHook._install_chromedriver):
+        (AutoinstallImportHook._install_geckodriver):
+        (AutoinstallImportHook.get_latest_pypi_url):
+        (AutoinstallImportHook.install_binary):
+        (autoinstall_everything):
+        (get_driver_filename):
+        (get_os_info):
+
+2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
+
         Add WebDriver support in browser driver part of BenchmarkRunner
         https://bugs.webkit.org/show_bug.cgi?id=174445
 
index b41a1c9..ffa2c68 100644 (file)
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from abc import ABCMeta, abstractmethod
+from webkitpy.benchmark_runner.utils import get_driver_binary_path
 
 
 class BrowserDriver(object):
@@ -32,3 +33,7 @@ class BrowserDriver(object):
     @abstractmethod
     def restore_env(self):
         pass
+
+    @property
+    def webdriver_binary_path(self):
+        return get_driver_binary_path(self.browser_name)
index 3c0207f..60c0fff 100644 (file)
@@ -49,6 +49,7 @@ class LinuxChromeDriver(LinuxBrowserDriver):
         if browser_build_path:
             binary_path = os.path.join(browser_build_path, 'chromium-browser')
             options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=options, executable_path=driver_executable)
         super(LinuxChromeDriver, self).launch_webdriver(url, driver)
         return driver
index bda2131..6bea172 100644 (file)
@@ -47,6 +47,7 @@ class LinuxFirefoxDriver(LinuxBrowserDriver):
         if browser_build_path:
             binary_path = os.path.join(browser_build_path, 'firefox-bin')
             options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=options, executable_path=driver_executable)
         super(LinuxFirefoxDriver, self).launch_webdriver(url, driver)
         return driver
index 12af135..013b100 100644 (file)
@@ -2,8 +2,6 @@
 
 import logging
 import os
-import subprocess
-import time
 
 from osx_browser_driver import OSXBrowserDriver
 from selenium import webdriver
@@ -34,7 +32,8 @@ class OSXChromeDriver(OSXBrowserDriver):
             app_path = os.path.join(browser_build_path, self.app_name)
             binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
             chrome_options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=chrome_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
 
@@ -54,6 +53,7 @@ class OSXChromeCanaryDriver(OSXBrowserDriver):
         app_path = os.path.join(browser_build_path, self.app_name)
         binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
         chrome_options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=chrome_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
index 8254cd4..67afe83 100644 (file)
@@ -2,8 +2,6 @@
 
 import logging
 import os
-import subprocess
-import time
 
 from osx_browser_driver import OSXBrowserDriver
 from selenium import webdriver
@@ -30,7 +28,8 @@ class OSXFirefoxDriver(OSXBrowserDriver):
             app_path = os.path.join(browser_build_path, self.app_name)
             binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
             firefox_options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=firefox_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
 
@@ -50,6 +49,7 @@ class OSXFirefoxNightlyDriver(OSXBrowserDriver):
         app_path = os.path.join(browser_build_path, self.app_name)
         binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
         firefox_options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=firefox_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
index 4f92a0f..9be0f44 100644 (file)
@@ -8,6 +8,8 @@ import signal
 import shutil
 import sys
 
+from webkitpy.common.memoized import memoized
+
 
 _log = logging.getLogger(__name__)
 
@@ -46,6 +48,20 @@ def force_remove(path):
         pass
 
 
+@memoized
+def get_driver_binary_path(browser_name):
+    if browser_name.startswith('chrome'):
+        import webkitpy.thirdparty.autoinstalled.chromedriver
+        driver_init_file = webkitpy.thirdparty.autoinstalled.chromedriver.__file__
+        driver_executable = os.path.join(os.path.dirname(os.path.realpath(driver_init_file)), 'chromedriver')
+        return driver_executable
+    elif browser_name.startswith('firefox'):
+        import webkitpy.thirdparty.autoinstalled.geckodriver
+        driver_init_file = webkitpy.thirdparty.autoinstalled.geckodriver.__file__
+        driver_executable = os.path.join(os.path.dirname(os.path.realpath(driver_init_file)), 'geckodriver')
+        return driver_executable
+
+
 def write_defaults(domain, key, value):
     # Returns whether the key in the domain is updated
     from Foundation import NSUserDefaults
index d0f7c57..84ca7b0 100644 (file)
@@ -275,7 +275,7 @@ class AutoInstaller(object):
             raise Exception(message)
 
         try:
-            self._extract_all(zip_file, scratch_dir)
+            self._extract_all(zip_file, target_path)
         finally:
             zip_file.close()
 
index 502aa55..425b0fb 100644 (file)
 
 
 import codecs
+import json
 import os
+import re
 import sys
+import urllib2
 
+from collections import namedtuple
+from distutils import spawn
 from webkitpy.common.system.autoinstall import AutoInstaller
 from webkitpy.common.system.filesystem import FileSystem
 
 _THIRDPARTY_DIR = os.path.dirname(__file__)
 _AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
 
+CHROME_DRIVER_URL = "http://chromedriver.storage.googleapis.com/"
+FIREFOX_RELEASES_URL = "https://api.github.com/repos/mozilla/geckodriver/releases"
+
 # Putting the autoinstall code into webkitpy/thirdparty/__init__.py
 # ensures that no autoinstalling occurs until a caller imports from
 # webkitpy.thirdparty.  This is useful if the caller wants to configure
@@ -46,6 +54,7 @@ _AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
 # We put auto-installed third-party modules in this directory--
 #
 #     webkitpy/thirdparty/autoinstalled
+
 fs = FileSystem()
 fs.maybe_make_directory(_AUTOINSTALLED_DIR)
 
@@ -91,6 +100,12 @@ class AutoinstallImportHook(object):
             self._install_keyring()
         elif '.twisted_15_5_0' in fullname:
             self._install_twisted_15_5_0()
+        elif '.selenium' in fullname:
+            self._install_selenium()
+        elif '.chromedriver' in fullname:
+            self._install_chromedriver()
+        elif '.geckodriver' in fullname:
+            self._install_geckodriver()
 
     def _install_mechanize(self):
         self._install("http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.5.tar.gz",
@@ -151,10 +166,45 @@ class AutoinstallImportHook(object):
         installer.install(url="https://pypi.python.org/packages/source/T/Twisted/Twisted-15.5.0.tar.bz2#md5=0831d7c90d0020062de0f7287530a285", url_subpath="Twisted-15.5.0/twisted")
         installer.install(url="https://pypi.python.org/packages/source/z/zope.interface/zope.interface-4.1.3.tar.gz#md5=9ae3d24c0c7415deb249dd1a132f0f79", url_subpath="zope.interface-4.1.3/src/zope")
 
+    def _install_selenium(self):
+        self._ensure_autoinstalled_dir_is_in_sys_path()
+        url, url_subpath = self.get_latest_pypi_url('selenium')
+        self._install(url=url, url_subpath=url_subpath)
+
+    def _install_chromedriver(self):
+        filename_postfix = get_driver_filename().chrome
+        if filename_postfix != "unsupported":
+            version = urllib2.urlopen(CHROME_DRIVER_URL + 'LATEST_RELEASE').read().strip()
+            full_chrome_url = "{base_url}{version}/chromedriver_{os}.zip".format(base_url=CHROME_DRIVER_URL, version=version, os=filename_postfix)
+            self.install_binary(full_chrome_url, 'chromedriver')
+
+    def _install_geckodriver(self):
+        filename_postfix = get_driver_filename().firefox
+        if filename_postfix != "unsupported":
+            firefox_releases_blob = urllib2.urlopen(FIREFOX_RELEASES_URL)
+            firefox_releases_line_separated = json.dumps(json.load(firefox_releases_blob), indent=0).strip()
+            all_firefox_release_urls = "\n".join(re.findall(r'.*browser_download_url.*', firefox_releases_line_separated))
+            full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
+            self.install_binary(full_firefox_url, 'geckodriver')
+
     def _install(self, url, url_subpath=None, target_name=None):
         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
         installer.install(url=url, url_subpath=url_subpath, target_name=target_name)
 
+    def get_latest_pypi_url(self, package_name, url_subpath_format='{name}-{version}/{lname}'):
+        json_url = "https://pypi.python.org/pypi/%s/json" % package_name
+        response = urllib2.urlopen(json_url)
+        data = json.load(response)
+        url = data['urls'][1]['url']
+        subpath = url_subpath_format.format(name=package_name, version=data['info']['version'], lname=package_name.lower())
+        return (url, subpath)
+
+    def install_binary(self, url, name):
+        self._install(url=url, target_name=name)
+        directory = os.path.join(_AUTOINSTALLED_DIR, name)
+        os.chmod(os.path.join(directory, name), 0755)
+        open(os.path.join(directory, '__init__.py'), 'w+').close()
+
 
 _hook = AutoinstallImportHook()
 sys.meta_path.append(_hook)
@@ -164,3 +214,21 @@ def autoinstall_everything():
     install_methods = [method for method in dir(_hook.__class__) if method.startswith('_install_')]
     for method in install_methods:
         getattr(_hook, method)()
+
+def get_driver_filename():
+    os_name, os_type = get_os_info()
+    chrome_os, filefox_os = 'unsupported', 'unsupported'
+    if os_name == 'Linux' and os_type == '64':
+        chrome_os, firefox_os = 'linux64', 'linux64'
+    elif os_name == 'Linux':
+        chrome_os, firefox_os = 'linux32', 'linux32'
+    elif os_name == 'Darwin':
+        chrome_os, firefox_os = 'mac64', 'macos'
+    DriverFilenameForBrowser = namedtuple('DriverFilenameForBrowser', ['chrome', 'firefox'])
+    return DriverFilenameForBrowser(chrome_os, firefox_os)
+
+def get_os_info():
+    import platform
+    os_name = platform.system()
+    os_type = platform.machine()[-2:]
+    return (os_name, os_type)