WebDriver: add support for test expectations
[WebKit-https.git] / Tools / Scripts / webkitpy / thirdparty / __init__.py
1 # Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 # This module is required for Python to treat this directory as a package.
24
25 """Autoinstalls third-party code required by WebKit."""
26
27
28 import codecs
29 import json
30 import os
31 import re
32 import sys
33 import urllib2
34
35 from collections import namedtuple
36 from distutils import spawn
37 from webkitpy.common.system.autoinstall import AutoInstaller
38 from webkitpy.common.system.filesystem import FileSystem
39
40 _THIRDPARTY_DIR = os.path.dirname(__file__)
41 _AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
42
43 CHROME_DRIVER_URL = "http://chromedriver.storage.googleapis.com/"
44 FIREFOX_RELEASES_URL = "https://api.github.com/repos/mozilla/geckodriver/releases"
45
46 # Putting the autoinstall code into webkitpy/thirdparty/__init__.py
47 # ensures that no autoinstalling occurs until a caller imports from
48 # webkitpy.thirdparty.  This is useful if the caller wants to configure
49 # logging prior to executing autoinstall code.
50
51 # FIXME: If any of these servers is offline, webkit-patch breaks (and maybe
52 # other scripts do, too). See <http://webkit.org/b/42080>.
53
54 # We put auto-installed third-party modules in this directory--
55 #
56 #     webkitpy/thirdparty/autoinstalled
57
58 fs = FileSystem()
59 fs.maybe_make_directory(_AUTOINSTALLED_DIR)
60
61 init_path = fs.join(_AUTOINSTALLED_DIR, "__init__.py")
62 if not fs.exists(init_path):
63     fs.write_text_file(init_path, "")
64
65 readme_path = fs.join(_AUTOINSTALLED_DIR, "README")
66 if not fs.exists(readme_path):
67     fs.write_text_file(readme_path,
68         "This directory is auto-generated by WebKit and is "
69         "safe to delete.\nIt contains needed third-party Python "
70         "packages automatically downloaded from the web.")
71
72
73 class AutoinstallImportHook(object):
74     def __init__(self, filesystem=None):
75         self._fs = filesystem or FileSystem()
76
77     def _ensure_autoinstalled_dir_is_in_sys_path(self):
78         # Some packages require that the are being put somewhere under a directory in sys.path.
79         if not _AUTOINSTALLED_DIR in sys.path:
80             sys.path.insert(0, _AUTOINSTALLED_DIR)
81
82     def find_module(self, fullname, _):
83         # This method will run before each import. See http://www.python.org/dev/peps/pep-0302/
84         if '.autoinstalled' not in fullname:
85             return
86
87         # Note: all of the methods must follow the "_install_XXX" convention in
88         # order for autoinstall_everything(), below, to work properly.
89         if '.mechanize' in fullname:
90             self._install_mechanize()
91         elif '.pep8' in fullname:
92             self._install_pep8()
93         elif '.pylint' in fullname:
94             self._install_pylint()
95         elif '.coverage' in fullname:
96             self._install_coverage()
97         elif '.buildbot' in fullname:
98             self._install_buildbot()
99         elif '.keyring' in fullname:
100             self._install_keyring()
101         elif '.twisted_15_5_0' in fullname:
102             self._install_twisted_15_5_0()
103         elif '.selenium' in fullname:
104             self._install_selenium()
105         elif '.chromedriver' in fullname:
106             self.install_chromedriver()
107         elif '.geckodriver' in fullname:
108             self.install_geckodriver()
109         elif '.mozlog' in fullname:
110             self._install_mozlog()
111         elif '.mozprocess' in fullname:
112             self._install_mozprocess()
113         elif '.pytest_timeout' in fullname:
114             self._install_pytest_timeout()
115         elif '.pytest' in fullname:
116             self._install_pytest()
117
118     def _install_mechanize(self):
119         self._install("https://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.5.tar.gz",
120                              "mechanize-0.2.5/mechanize")
121
122     def _install_keyring(self):
123         self._install("https://pypi.python.org/packages/7d/a9/8c6bf60710781ce13a9987c0debda8adab35eb79c6b5525f7fe5240b7a8a/keyring-7.3.1.tar.gz#md5=99dd793e1233964eb87cf56406ee66f6",
124                              "keyring-7.3.1/keyring")
125
126     def _install_pep8(self):
127         self._install("https://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
128                              "pep8-0.5.0/pep8.py")
129
130     def _install_mozlog(self):
131         self._ensure_autoinstalled_dir_is_in_sys_path()
132         self._install("https://pypi.python.org/packages/10/d5/d286b5dc3f40e32d2a9b3cab0b5b20a05d704958b44b4c5a9aed6472deab/mozlog-3.5.tar.gz#md5=3282c70e7037266f83d8c80119129b75",
133                               "mozlog-3.5/mozlog")
134
135     def _install_mozprocess(self):
136         self._ensure_autoinstalled_dir_is_in_sys_path()
137         self._install("https://pypi.python.org/packages/cb/26/144dbc28d1f40e392f8a0cbee771ba624a61017f016c77ad88424d84b230/mozprocess-0.25.tar.gz#md5=07a04e6ae1a705705e4b44969fe7a182",
138                               "mozprocess-0.25/mozprocess")
139
140     def _install_pytest_timeout(self):
141         self._install("https://pypi.python.org/packages/cc/b7/b2a61365ea6b6d2e8881360ae7ed8dad0327ad2df89f2f0be4a02304deb2/pytest-timeout-1.2.0.tar.gz#md5=83607d91aa163562c7ee835da57d061d",
142                               "pytest-timeout-1.2.0/pytest_timeout.py")
143
144     def _install_pytest(self):
145         self._install("https://pypi.python.org/packages/90/e3/e075127d39d35f09a500ebb4a90afd10f9ef0a1d28a6d09abeec0e444fdd/py-1.5.2.tar.gz#md5=279ca69c632069e1b71e11b14641ca28",
146                               "py-1.5.2/py")
147         self._install("https://pypi.python.org/packages/1f/f8/8cd74c16952163ce0db0bd95fdd8810cbf093c08be00e6e665ebf0dc3138/pytest-3.2.5.tar.gz#md5=6dbe9bb093883f75394a689a1426ac6f",
148                               "pytest-3.2.5/_pytest")
149         self._install("https://pypi.python.org/packages/1f/f8/8cd74c16952163ce0db0bd95fdd8810cbf093c08be00e6e665ebf0dc3138/pytest-3.2.5.tar.gz#md5=6dbe9bb093883f75394a689a1426ac6f",
150                               "pytest-3.2.5/pytest.py")
151
152     def _install_pylint(self):
153         self._ensure_autoinstalled_dir_is_in_sys_path()
154         if (not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "pylint")) or
155             not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "logilab/astng")) or
156             not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "logilab/common"))):
157             installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
158             files_to_remove = []
159             if sys.platform == 'win32':
160                 files_to_remove = ['test/data/write_protected_file.txt']
161             installer.install("https://pypi.python.org/packages/source/l/logilab-common/logilab-common-0.58.1.tar.gz#md5=77298ab2d8bb8b4af9219791e7cee8ce", url_subpath="logilab-common-0.58.1", target_name="logilab/common", files_to_remove=files_to_remove)
162             installer.install("https://pypi.python.org/packages/source/l/logilab-astng/logilab-astng-0.24.1.tar.gz#md5=ddaf66e4d85714d9c47a46d4bed406de", url_subpath="logilab-astng-0.24.1", target_name="logilab/astng")
163             installer.install('https://pypi.python.org/packages/source/p/pylint/pylint-0.25.1.tar.gz#md5=728bbc2b339bc3749af013709a7f87a5', url_subpath="pylint-0.25.1", target_name="pylint")
164
165     # autoinstalled.buildbot is used by BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
166     # and should ideally match the version of BuildBot used at build.webkit.org.
167     def _install_buildbot(self):
168         # The buildbot package uses jinja2, for example, in buildbot/status/web/base.py.
169         # buildbot imports jinja2 directly (as though it were installed on the system),
170         # so the search path needs to include jinja2.  We put jinja2 in
171         # its own directory so that we can include it in the search path
172         # without including other modules as a side effect.
173         jinja_dir = self._fs.join(_AUTOINSTALLED_DIR, "jinja2")
174         installer = AutoInstaller(append_to_search_path=True, target_dir=jinja_dir)
175         installer.install(url="https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.6.tar.gz#md5=1c49a8825c993bfdcf55bb36897d28a2",
176                                                 url_subpath="Jinja2-2.6/jinja2")
177
178         SQLAlchemy_dir = self._fs.join(_AUTOINSTALLED_DIR, "sqlalchemy")
179         installer = AutoInstaller(append_to_search_path=True, target_dir=SQLAlchemy_dir)
180         installer.install(url="https://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.7.7.tar.gz#md5=ddf6df7e014cea318fa981364f3f93b9",
181                                                  url_subpath="SQLAlchemy-0.7.7/lib/sqlalchemy")
182
183         twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted")
184         installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
185         installer.install(url="https://pypi.python.org/packages/source/T/Twisted/Twisted-12.1.0.tar.bz2#md5=f396f1d6f5321e869c2f89b2196a9eb5", url_subpath="Twisted-12.1.0/twisted")
186
187         self._install("https://pypi.python.org/packages/source/b/buildbot/buildbot-0.8.6p1.tar.gz#md5=b6727d2810c692062c657492bcbeac6a", "buildbot-0.8.6p1/buildbot")
188
189     def _install_coverage(self):
190         self._ensure_autoinstalled_dir_is_in_sys_path()
191         self._install(url="https://pypi.python.org/packages/source/c/coverage/coverage-3.5.1.tar.gz#md5=410d4c8155a4dab222f2bc51212d4a24", url_subpath="coverage-3.5.1/coverage")
192
193     def _install_twisted_15_5_0(self):
194         twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted_15_5_0")
195         installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
196         installer.install(url="https://pypi.python.org/packages/source/T/Twisted/Twisted-15.5.0.tar.bz2#md5=0831d7c90d0020062de0f7287530a285", url_subpath="Twisted-15.5.0/twisted")
197         installer.install(url="https://pypi.python.org/packages/source/z/zope.interface/zope.interface-4.1.3.tar.gz#md5=9ae3d24c0c7415deb249dd1a132f0f79", url_subpath="zope.interface-4.1.3/src/zope")
198
199     @staticmethod
200     def greater_than_equal_to_version(minimum, version):
201         for i in xrange(len(minimum.split('.'))):
202             if int(version.split('.')[i]) > int(minimum.split('.')[i]):
203                 return True
204             if int(version.split('.')[i]) < int(minimum.split('.')[i]):
205                 return False
206         return True
207
208     def _install_selenium(self):
209         self._ensure_autoinstalled_dir_is_in_sys_path()
210         try:
211             url, url_subpath = self.get_latest_pypi_url('selenium')
212         except urllib2.URLError:
213             minimum_version = '3.5.0'
214             if os.path.isfile(os.path.join(_AUTOINSTALLED_DIR, 'selenium', '__init__.py')):
215                 import selenium.webdriver
216                 if AutoinstallImportHook.greater_than_equal_to_version(minimum_version, selenium.webdriver.__version__):
217                     sys.stderr.write('\nFailed to find latest selenium, falling back to existing {} version\n'.format(selenium.webdriver.__version__))
218                     return
219
220             # URL for installing the minimum required version.
221             url = 'https://pypi.python.org/packages/ac/d7/1928416439d066c60f26c87a8d1b78a8edd64c7d05a0aa917fa97a8ee02d/selenium-3.5.0.tar.gz#986702fdd0e2aec6a4eda134678b8b3f'
222             url_subpath = 'selenium-{}/selenium'.format(minimum_version)
223             sys.stderr.write('\nFailed to find latest selenium, falling back to minimum {} version\n'.format(minimum_version))
224         self._install(url=url, url_subpath=url_subpath)
225
226     def install_chromedriver(self):
227         filename_postfix = get_driver_filename().chrome
228         if filename_postfix != "unsupported":
229             version = urllib2.urlopen(CHROME_DRIVER_URL + 'LATEST_RELEASE').read().strip()
230             full_chrome_url = "{base_url}{version}/chromedriver_{os}.zip".format(base_url=CHROME_DRIVER_URL, version=version, os=filename_postfix)
231             self.install_binary(full_chrome_url, 'chromedriver')
232
233     def install_geckodriver(self):
234         filename_postfix = get_driver_filename().firefox
235         if filename_postfix != "unsupported":
236             firefox_releases_blob = urllib2.urlopen(FIREFOX_RELEASES_URL)
237             firefox_releases_line_separated = json.dumps(json.load(firefox_releases_blob), indent=0).strip()
238             all_firefox_release_urls = "\n".join(re.findall(r'.*browser_download_url.*', firefox_releases_line_separated))
239             full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
240             self.install_binary(full_firefox_url, 'geckodriver')
241
242     def _install(self, url, url_subpath=None, target_name=None):
243         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
244         installer.install(url=url, url_subpath=url_subpath, target_name=target_name)
245
246     def get_latest_pypi_url(self, package_name, url_subpath_format='{name}-{version}/{lname}'):
247         json_url = "https://pypi.python.org/pypi/%s/json" % package_name
248         response = urllib2.urlopen(json_url)
249         data = json.load(response)
250         url = data['urls'][1]['url']
251         subpath = url_subpath_format.format(name=package_name, version=data['info']['version'], lname=package_name.lower())
252         return (url, subpath)
253
254     def install_binary(self, url, name):
255         self._install(url=url, target_name=name)
256         directory = os.path.join(_AUTOINSTALLED_DIR, name)
257         os.chmod(os.path.join(directory, name), 0755)
258         open(os.path.join(directory, '__init__.py'), 'w+').close()
259
260
261 _hook = AutoinstallImportHook()
262 sys.meta_path.append(_hook)
263
264
265 def autoinstall_everything():
266     install_methods = [method for method in dir(_hook.__class__) if method.startswith('_install_')]
267     for method in install_methods:
268         getattr(_hook, method)()
269
270 def get_driver_filename():
271     os_name, os_type = get_os_info()
272     chrome_os, filefox_os = 'unsupported', 'unsupported'
273     if os_name == 'Linux' and os_type == '64':
274         chrome_os, firefox_os = 'linux64', 'linux64'
275     elif os_name == 'Linux':
276         chrome_os, firefox_os = 'linux32', 'linux32'
277     elif os_name == 'Darwin':
278         chrome_os, firefox_os = 'mac64', 'macos'
279     DriverFilenameForBrowser = namedtuple('DriverFilenameForBrowser', ['chrome', 'firefox'])
280     return DriverFilenameForBrowser(chrome_os, firefox_os)
281
282 def get_os_info():
283     import platform
284     os_name = platform.system()
285     os_type = platform.machine()[-2:]
286     return (os_name, os_type)