pytest is not correctly auto-installed
[WebKit-https.git] / Tools / Scripts / generate-jsc-bundle
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2018 Igalia S.L.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are met:
7 #
8 # 1. Redistributions of source code must retain the above copyright notice, this
9 #    list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright notice,
11 #    this list of conditions and the following disclaimer in the documentation
12 #    and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
18 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24
25 import base64
26 import datetime
27 import hashlib
28 import json
29 import optparse
30 import os
31 import shutil
32 import subprocess
33 import sys
34 import tempfile
35 import zipfile
36
37 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
38 sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'flatpak'))
39 sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'jhbuild'))
40 import jhbuildutils
41 import flatpakutils
42
43
44 # Ideally we should use something like lddtree or create our own version of that
45 # But in practice for jsc bundles there isn't recursive library entries, so we
46 # use standard ldd here just to avoid having to require lddtree.
47 def ldd_get_libs_and_interpreter(binary):
48     lddCommand = ['ldd', binary]
49     lddProcess = subprocess.Popen(lddCommand, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
50     stdout, stderr = lddProcess.communicate()
51     if lddProcess.returncode != 0:
52         raise RuntimeError('The ldd command returned non-zero status')
53     libs = []
54     for line in stdout.splitlines():
55         line = line.strip()
56         if '=>' in line:
57             line = line.split('=>')[1].strip()
58             if 'not found' in line:
59                 raise RuntimeError('Some dependencies can not be found with ldd.')
60             line = line.split(' ')[0].strip()
61             if os.path.isfile(line):
62                 libs.append(line)
63         else:
64             line = line.split(' ')[0].strip()
65             if os.path.isfile(line):
66                 interpreter = line
67     return libs, interpreter
68
69
70 def generate_readme(bundleTmpDir, builderName, configuration, platform, revision):
71     print('Generate README.txt file')
72     readmeFile = os.path.join(bundleTmpDir, 'README.txt')
73     with open(readmeFile, 'w') as readmeHandle:
74         readmeHandle.write('JSC bundle details\n')
75         readmeHandle.write(' Builder name: %s\n' % builderName)
76         readmeHandle.write(' Builder date: %s\n' % datetime.datetime.now().isoformat())
77         readmeHandle.write(' Configuration: %s\n' % configuration)
78         readmeHandle.write(' WebKit Platform: %s\n' % platform)
79         readmeHandle.write(' WebKit Revision: %s\n' % revision)
80         readmeHandle.write('\nInstructions: Execute the run-jsc wrapper script.\n')
81     return True
82
83
84 def generate_wrapper_script(bundleTmpDir, interpreter):
85     print('Generate wrapper script run-jsc')
86     scriptFile = os.path.join(bundleTmpDir, 'run-jsc')
87     with open(scriptFile, 'w') as scriptHandle:
88         scriptHandle.write('#!/bin/sh\n')
89         scriptHandle.write('MYDIR="$(dirname $(readlink -f $0))"\n')
90         scriptHandle.write('export LD_LIBRARY_PATH="${MYDIR}/lib"\n')
91         scriptHandle.write('exec "${MYDIR}/lib/%s" "${MYDIR}/bin/jsc" "$@"\n' % os.path.basename(interpreter))
92     os.chmod(scriptFile, 0755)
93
94
95 def copy_and_remove_rpath(origFile, destinationDir, type='bin'):
96     if not os.path.isfile(origFile):
97         raise ValueError('Can not find file %s' % origFile)
98     print('Copy to bundle [%s]: %s' % (type, origFile))
99     shutil.copy(origFile, destinationDir)
100     try:
101         patchElfCommand = ['patchelf', '--remove-rpath', os.path.join(destinationDir, os.path.basename(origFile))]
102         if subprocess.call(patchElfCommand) != 0:
103             print('WARNING: The patchelf command returned non-zero status')
104     except OSError as e:
105         if e.errno == os.errno.ENOENT:
106             print('WARNING: patchelf not found. Not modifying rpath')
107         else:
108             raise
109
110
111 def createJSCBundle(configuration, revision=None, builderName=None, platform=None):
112     buildDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'WebKitBuild'))
113     binDir = os.path.join(buildDir, configuration.capitalize(), 'bin')
114     libDir = os.path.join(buildDir, configuration.capitalize(), 'lib')
115     jscBinary = os.path.join(binDir, 'jsc')
116     if not os.path.isfile(jscBinary) or not os.access(jscBinary, os.X_OK):
117         raise ValueError('Cannot find jsc at %s' % jscBinary)
118
119     # Define names and paths for the generation of the bundle.
120     bundleTmpDir = os.path.join(buildDir, 'jsc_tmp')
121     bundleTmpLibDir = os.path.join(bundleTmpDir, 'lib')
122     bundleTmpBinDir = os.path.join(bundleTmpDir, 'bin')
123     bundleFileName = 'jsc_' + configuration
124     bundleFileCompressed = os.path.join(buildDir, bundleFileName + '.zip')
125
126     # Clean everything from previous runs
127     if os.path.isdir(bundleTmpDir):
128         shutil.rmtree(bundleTmpDir)
129     if os.path.isfile(bundleFileCompressed):
130         os.remove(bundleFileCompressed)
131
132     # Create bundleTmpDir and put there everything needed.
133     os.makedirs(bundleTmpDir)
134     os.makedirs(bundleTmpLibDir)
135     os.makedirs(bundleTmpBinDir)
136     copy_and_remove_rpath(jscBinary, bundleTmpBinDir, type='bin')
137     libraries, interpreter = ldd_get_libs_and_interpreter(jscBinary)
138     copy_and_remove_rpath(interpreter, bundleTmpLibDir, type='interpreter')
139     for library in libraries:
140         copy_and_remove_rpath(library, bundleTmpLibDir, type='lib')
141     generate_readme(bundleTmpDir, builderName, configuration, platform, revision)
142     generate_wrapper_script(bundleTmpDir, interpreter)
143
144     # jsvu project prefers .zip rather than .tar.xz
145     with zipfile.ZipFile(bundleFileCompressed, 'w', compression=zipfile.ZIP_DEFLATED) as zipHandle:
146         for dirname, subdirs, files in os.walk(bundleTmpDir):
147             for filename in files:
148                 systemFilePath = os.path.join(dirname, filename)
149                 zipFilePath = systemFilePath.replace(bundleTmpDir, '', 1).lstrip('/')
150                 zipHandle.write(systemFilePath, zipFilePath)
151
152     if not os.path.isfile(bundleFileCompressed):
153         raise RuntimeError('Unable to create the file %s' % bundleFileCompressed)
154     return bundleFileCompressed
155
156
157 def sha256sum(bundleFilePath):
158     hash = hashlib.sha256()
159     with open(bundleFilePath, 'rb') as f:
160         for chunk in iter(lambda: f.read(4096), b''):
161             hash.update(chunk)
162     return hash.hexdigest()
163
164
165 # The expected format for --remote-config-file is something like:
166 # {
167 # "servername": "webkitgtk.org",
168 # "serveraddress": "webkitgtk.intranet-address.local",
169 # "serverport": "23",
170 # "username": "upload-bot-64",
171 # "baseurl": "https://webkitgtk.org/jsc-built-products/x86_64",
172 # "sshkey": "output of the priv key in base64. E.g. cat ~/.ssh/id_rsa|base64 -w0"
173 # }
174 def uploadJSCBundle(bundleFilePath, remoteConfigFile, configuration, revision):
175     remoteData = json.load(open(remoteConfigFile))
176     remoteFileName = str(revision) + '.zip'
177     remoteFileBundlePathName = os.path.join(configuration, remoteFileName)
178     remoteFileHashPathName = os.path.join(configuration, str(revision) + '.sha256sum')
179     with tempfile.NamedTemporaryFile() as sshkeyfile:
180         # In theory NamedTemporaryFile() is already created 0600. But it don't hurts ensuring this again here.
181         os.chmod(sshkeyfile.name, 0600)
182         sshkeyfile.write(base64.b64decode(remoteData['sshkey']))
183         sshkeyfile.flush()
184         # Generate and upload also a sha256 hash
185         with tempfile.NamedTemporaryFile() as hashcheckfile:
186             hashforbundle = sha256sum(bundleFilePath)
187             os.chmod(hashcheckfile.name, 0644)
188             hashcheckfile.write('%s %s\n' % (hashforbundle, remoteFileName))
189             hashcheckfile.flush()
190             with tempfile.NamedTemporaryFile() as uploadinstructionsfile:
191                 uploadinstructionsfile.write('progress\n')
192                 uploadinstructionsfile.write('put %s %s\n' % (bundleFilePath, remoteFileBundlePathName))
193                 uploadinstructionsfile.write('put %s %s\n' % (hashcheckfile.name, remoteFileHashPathName))
194                 uploadinstructionsfile.write('quit\n')
195                 uploadinstructionsfile.flush()
196                 # The idea of this is to ensure scp doesn't ask any question (not even on the first run).
197                 # This should be secure enough according to https://www.gremwell.com/ssh-mitm-public-key-authentication
198                 sftpCommand = ['sftp',
199                                '-o', 'StrictHostKeyChecking=no',
200                                '-o', 'UserKnownHostsFile=/dev/null',
201                                '-o', 'LogLevel=ERROR',
202                                '-P', remoteData['serverport'],
203                                '-i', sshkeyfile.name,
204                                '-b', uploadinstructionsfile.name,
205                                '%s@%s' % (remoteData['username'], remoteData['serveraddress'])]
206                 print('Uploading bundle to %s as %s with sha256 hash %s' % (remoteData['servername'], remoteFileBundlePathName, hashforbundle))
207                 if subprocess.call(sftpCommand) != 0:
208                     raise RuntimeError('The sftp command returned non-zero status')
209
210     print('Done: archive sucesfully uploaded to %s/%s' % (remoteData['baseurl'], remoteFileBundlePathName))
211     return 0
212
213
214 def main():
215     parser = optparse.OptionParser('usage: %prog [options]')
216     parser.add_option('--platform', dest='platform')
217     parser.add_option('--debug', action='store_const', const='debug', dest='configuration')
218     parser.add_option('--release', action='store_const', const='release', dest='configuration')
219     parser.add_option('--revision', action='store', type='string', dest='revision')
220     parser.add_option('--builder-name', action='store', type='string', dest='buildername')
221     parser.add_option('--remote-config-file', action='store',  type='string', dest='remoteConfigFile')
222     options, args = parser.parse_args()
223
224     if not options.platform:
225         parser.error('Platform is required')
226         return 1
227     if not options.configuration:
228         parser.error('Configuration is required')
229         return 1
230
231     platform = options.platform.lower()
232     configuration = options.configuration.lower()
233     if platform == 'gtk':
234         flatpakutils.run_in_sandbox_if_available(sys.argv)
235         if not flatpakutils.is_sandboxed():
236             jhbuildutils.enter_jhbuild_environment_if_available("gtk")
237     else:
238         raise NotImplementedError('Unsupported platform')
239
240     bundleFilePath = createJSCBundle(configuration, options.revision, options.buildername, platform)
241     print('Bundle file created at: %s' % bundleFilePath)
242     if options.remoteConfigFile is not None and os.path.isfile(options.remoteConfigFile):
243         return uploadJSCBundle(bundleFilePath, options.remoteConfigFile, options.configuration, options.revision)
244     return 0
245
246
247 if __name__ == '__main__':
248     sys.exit(main())