WebDriver: add timeout option to run-webdriver-tests script
[WebKit-https.git] / Tools / Scripts / webkitpy / thirdparty / __init__.py
1 # Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 # This module is required for Python to treat this directory as a package.
24
25 """Autoinstalls third-party code required by WebKit."""
26
27
28 import codecs
29 import json
30 import os
31 import re
32 import sys
33 import urllib2
34
35 from collections import namedtuple
36 from distutils import spawn
37 from webkitpy.common.system.autoinstall import AutoInstaller
38 from webkitpy.common.system.filesystem import FileSystem
39
40 _THIRDPARTY_DIR = os.path.dirname(__file__)
41 _AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
42
43 CHROME_DRIVER_URL = "http://chromedriver.storage.googleapis.com/"
44 FIREFOX_RELEASES_URL = "https://api.github.com/repos/mozilla/geckodriver/releases"
45
46 # Putting the autoinstall code into webkitpy/thirdparty/__init__.py
47 # ensures that no autoinstalling occurs until a caller imports from
48 # webkitpy.thirdparty.  This is useful if the caller wants to configure
49 # logging prior to executing autoinstall code.
50
51 # FIXME: If any of these servers is offline, webkit-patch breaks (and maybe
52 # other scripts do, too). See <http://webkit.org/b/42080>.
53
54 # We put auto-installed third-party modules in this directory--
55 #
56 #     webkitpy/thirdparty/autoinstalled
57
58 fs = FileSystem()
59 fs.maybe_make_directory(_AUTOINSTALLED_DIR)
60
61 init_path = fs.join(_AUTOINSTALLED_DIR, "__init__.py")
62 if not fs.exists(init_path):
63     fs.write_text_file(init_path, "")
64
65 readme_path = fs.join(_AUTOINSTALLED_DIR, "README")
66 if not fs.exists(readme_path):
67     fs.write_text_file(readme_path,
68         "This directory is auto-generated by WebKit and is "
69         "safe to delete.\nIt contains needed third-party Python "
70         "packages automatically downloaded from the web.")
71
72
73 class AutoinstallImportHook(object):
74     def __init__(self, filesystem=None):
75         self._fs = filesystem or FileSystem()
76
77     def _ensure_autoinstalled_dir_is_in_sys_path(self):
78         # Some packages require that the are being put somewhere under a directory in sys.path.
79         if not _AUTOINSTALLED_DIR in sys.path:
80             sys.path.insert(0, _AUTOINSTALLED_DIR)
81
82     def find_module(self, fullname, _):
83         # This method will run before each import. See http://www.python.org/dev/peps/pep-0302/
84         if '.autoinstalled' not in fullname:
85             return
86
87         # Note: all of the methods must follow the "_install_XXX" convention in
88         # order for autoinstall_everything(), below, to work properly.
89         if '.mechanize' in fullname:
90             self._install_mechanize()
91         elif '.pep8' in fullname:
92             self._install_pep8()
93         elif '.pylint' in fullname:
94             self._install_pylint()
95         elif '.coverage' in fullname:
96             self._install_coverage()
97         elif '.buildbot' in fullname:
98             self._install_buildbot()
99         elif '.keyring' in fullname:
100             self._install_keyring()
101         elif '.twisted_15_5_0' in fullname:
102             self._install_twisted_15_5_0()
103         elif '.selenium' in fullname:
104             self._install_selenium()
105         elif '.chromedriver' in fullname:
106             self.install_chromedriver()
107         elif '.geckodriver' in fullname:
108             self.install_geckodriver()
109         elif '.mozlog' in fullname:
110             self._install_mozlog()
111         elif '.mozprocess' in fullname:
112             self._install_mozprocess()
113         elif '.pytest_timeout' in fullname:
114             self._install_pytest_timeout()
115
116     def _install_mechanize(self):
117         self._install("https://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.5.tar.gz",
118                              "mechanize-0.2.5/mechanize")
119
120     def _install_keyring(self):
121         self._install("https://pypi.python.org/packages/7d/a9/8c6bf60710781ce13a9987c0debda8adab35eb79c6b5525f7fe5240b7a8a/keyring-7.3.1.tar.gz#md5=99dd793e1233964eb87cf56406ee66f6",
122                              "keyring-7.3.1/keyring")
123
124     def _install_pep8(self):
125         self._install("https://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
126                              "pep8-0.5.0/pep8.py")
127
128     def _install_mozlog(self):
129         self._ensure_autoinstalled_dir_is_in_sys_path()
130         self._install("https://pypi.python.org/packages/10/d5/d286b5dc3f40e32d2a9b3cab0b5b20a05d704958b44b4c5a9aed6472deab/mozlog-3.5.tar.gz#md5=3282c70e7037266f83d8c80119129b75",
131                               "mozlog-3.5/mozlog")
132
133     def _install_mozprocess(self):
134         self._ensure_autoinstalled_dir_is_in_sys_path()
135         self._install("https://pypi.python.org/packages/cb/26/144dbc28d1f40e392f8a0cbee771ba624a61017f016c77ad88424d84b230/mozprocess-0.25.tar.gz#md5=07a04e6ae1a705705e4b44969fe7a182",
136                               "mozprocess-0.25/mozprocess")
137
138     def _install_pytest_timeout(self):
139         self._install("https://pypi.python.org/packages/cc/b7/b2a61365ea6b6d2e8881360ae7ed8dad0327ad2df89f2f0be4a02304deb2/pytest-timeout-1.2.0.tar.gz#md5=83607d91aa163562c7ee835da57d061d",
140                               "pytest-timeout-1.2.0/pytest_timeout.py")
141
142     def _install_pylint(self):
143         self._ensure_autoinstalled_dir_is_in_sys_path()
144         if (not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "pylint")) or
145             not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "logilab/astng")) or
146             not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "logilab/common"))):
147             installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
148             files_to_remove = []
149             if sys.platform == 'win32':
150                 files_to_remove = ['test/data/write_protected_file.txt']
151             installer.install("https://pypi.python.org/packages/source/l/logilab-common/logilab-common-0.58.1.tar.gz#md5=77298ab2d8bb8b4af9219791e7cee8ce", url_subpath="logilab-common-0.58.1", target_name="logilab/common", files_to_remove=files_to_remove)
152             installer.install("https://pypi.python.org/packages/source/l/logilab-astng/logilab-astng-0.24.1.tar.gz#md5=ddaf66e4d85714d9c47a46d4bed406de", url_subpath="logilab-astng-0.24.1", target_name="logilab/astng")
153             installer.install('https://pypi.python.org/packages/source/p/pylint/pylint-0.25.1.tar.gz#md5=728bbc2b339bc3749af013709a7f87a5', url_subpath="pylint-0.25.1", target_name="pylint")
154
155     # autoinstalled.buildbot is used by BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
156     # and should ideally match the version of BuildBot used at build.webkit.org.
157     def _install_buildbot(self):
158         # The buildbot package uses jinja2, for example, in buildbot/status/web/base.py.
159         # buildbot imports jinja2 directly (as though it were installed on the system),
160         # so the search path needs to include jinja2.  We put jinja2 in
161         # its own directory so that we can include it in the search path
162         # without including other modules as a side effect.
163         jinja_dir = self._fs.join(_AUTOINSTALLED_DIR, "jinja2")
164         installer = AutoInstaller(append_to_search_path=True, target_dir=jinja_dir)
165         installer.install(url="https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.6.tar.gz#md5=1c49a8825c993bfdcf55bb36897d28a2",
166                                                 url_subpath="Jinja2-2.6/jinja2")
167
168         SQLAlchemy_dir = self._fs.join(_AUTOINSTALLED_DIR, "sqlalchemy")
169         installer = AutoInstaller(append_to_search_path=True, target_dir=SQLAlchemy_dir)
170         installer.install(url="https://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.7.7.tar.gz#md5=ddf6df7e014cea318fa981364f3f93b9",
171                                                  url_subpath="SQLAlchemy-0.7.7/lib/sqlalchemy")
172
173         twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted")
174         installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
175         installer.install(url="https://pypi.python.org/packages/source/T/Twisted/Twisted-12.1.0.tar.bz2#md5=f396f1d6f5321e869c2f89b2196a9eb5", url_subpath="Twisted-12.1.0/twisted")
176
177         self._install("https://pypi.python.org/packages/source/b/buildbot/buildbot-0.8.6p1.tar.gz#md5=b6727d2810c692062c657492bcbeac6a", "buildbot-0.8.6p1/buildbot")
178
179     def _install_coverage(self):
180         self._ensure_autoinstalled_dir_is_in_sys_path()
181         self._install(url="https://pypi.python.org/packages/source/c/coverage/coverage-3.5.1.tar.gz#md5=410d4c8155a4dab222f2bc51212d4a24", url_subpath="coverage-3.5.1/coverage")
182
183     def _install_twisted_15_5_0(self):
184         twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted_15_5_0")
185         installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
186         installer.install(url="https://pypi.python.org/packages/source/T/Twisted/Twisted-15.5.0.tar.bz2#md5=0831d7c90d0020062de0f7287530a285", url_subpath="Twisted-15.5.0/twisted")
187         installer.install(url="https://pypi.python.org/packages/source/z/zope.interface/zope.interface-4.1.3.tar.gz#md5=9ae3d24c0c7415deb249dd1a132f0f79", url_subpath="zope.interface-4.1.3/src/zope")
188
189     @staticmethod
190     def greater_than_equal_to_version(minimum, version):
191         for i in xrange(len(minimum.split('.'))):
192             if int(version.split('.')[i]) > int(minimum.split('.')[i]):
193                 return True
194             if int(version.split('.')[i]) < int(minimum.split('.')[i]):
195                 return False
196         return True
197
198     def _install_selenium(self):
199         self._ensure_autoinstalled_dir_is_in_sys_path()
200         try:
201             url, url_subpath = self.get_latest_pypi_url('selenium')
202         except urllib2.URLError:
203             minimum_version = '3.5.0'
204             if os.path.isfile(os.path.join(_AUTOINSTALLED_DIR, 'selenium', '__init__.py')):
205                 import selenium.webdriver
206                 if AutoinstallImportHook.greater_than_equal_to_version(minimum_version, selenium.webdriver.__version__):
207                     sys.stderr.write('\nFailed to find latest selenium, falling back to existing {} version\n'.format(selenium.webdriver.__version__))
208                     return
209
210             # URL for installing the minimum required version.
211             url = 'https://pypi.python.org/packages/ac/d7/1928416439d066c60f26c87a8d1b78a8edd64c7d05a0aa917fa97a8ee02d/selenium-3.5.0.tar.gz#986702fdd0e2aec6a4eda134678b8b3f'
212             url_subpath = 'selenium-{}/selenium'.format(minimum_version)
213             sys.stderr.write('\nFailed to find latest selenium, falling back to minimum {} version\n'.format(minimum_version))
214         self._install(url=url, url_subpath=url_subpath)
215
216     def install_chromedriver(self):
217         filename_postfix = get_driver_filename().chrome
218         if filename_postfix != "unsupported":
219             version = urllib2.urlopen(CHROME_DRIVER_URL + 'LATEST_RELEASE').read().strip()
220             full_chrome_url = "{base_url}{version}/chromedriver_{os}.zip".format(base_url=CHROME_DRIVER_URL, version=version, os=filename_postfix)
221             self.install_binary(full_chrome_url, 'chromedriver')
222
223     def install_geckodriver(self):
224         filename_postfix = get_driver_filename().firefox
225         if filename_postfix != "unsupported":
226             firefox_releases_blob = urllib2.urlopen(FIREFOX_RELEASES_URL)
227             firefox_releases_line_separated = json.dumps(json.load(firefox_releases_blob), indent=0).strip()
228             all_firefox_release_urls = "\n".join(re.findall(r'.*browser_download_url.*', firefox_releases_line_separated))
229             full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
230             self.install_binary(full_firefox_url, 'geckodriver')
231
232     def _install(self, url, url_subpath=None, target_name=None):
233         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
234         installer.install(url=url, url_subpath=url_subpath, target_name=target_name)
235
236     def get_latest_pypi_url(self, package_name, url_subpath_format='{name}-{version}/{lname}'):
237         json_url = "https://pypi.python.org/pypi/%s/json" % package_name
238         response = urllib2.urlopen(json_url)
239         data = json.load(response)
240         url = data['urls'][1]['url']
241         subpath = url_subpath_format.format(name=package_name, version=data['info']['version'], lname=package_name.lower())
242         return (url, subpath)
243
244     def install_binary(self, url, name):
245         self._install(url=url, target_name=name)
246         directory = os.path.join(_AUTOINSTALLED_DIR, name)
247         os.chmod(os.path.join(directory, name), 0755)
248         open(os.path.join(directory, '__init__.py'), 'w+').close()
249
250
251 _hook = AutoinstallImportHook()
252 sys.meta_path.append(_hook)
253
254
255 def autoinstall_everything():
256     install_methods = [method for method in dir(_hook.__class__) if method.startswith('_install_')]
257     for method in install_methods:
258         getattr(_hook, method)()
259
260 def get_driver_filename():
261     os_name, os_type = get_os_info()
262     chrome_os, filefox_os = 'unsupported', 'unsupported'
263     if os_name == 'Linux' and os_type == '64':
264         chrome_os, firefox_os = 'linux64', 'linux64'
265     elif os_name == 'Linux':
266         chrome_os, firefox_os = 'linux32', 'linux32'
267     elif os_name == 'Darwin':
268         chrome_os, firefox_os = 'mac64', 'macos'
269     DriverFilenameForBrowser = namedtuple('DriverFilenameForBrowser', ['chrome', 'firefox'])
270     return DriverFilenameForBrowser(chrome_os, firefox_os)
271
272 def get_os_info():
273     import platform
274     os_name = platform.system()
275     os_type = platform.machine()[-2:]
276     return (os_name, os_type)