[build.webkit.org] Rename mastercfg_unittest to steps_unittest
[WebKit-https.git] / Tools / BuildSlaveSupport / built-product-archive
1 #!/usr/bin/python
2
3 # Copyright (C) 2009, 2015 Apple Inc.  All rights reserved.
4 # Copyright (C) 2012 Google Inc. All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions
8 # are met:
9 #
10 # 1.  Redistributions of source code must retain the above copyright
11 #     notice, this list of conditions and the following disclaimer. 
12 # 2.  Redistributions in binary form must reproduce the above copyright
13 #     notice, this list of conditions and the following disclaimer in the
14 #     documentation and/or other materials provided with the distribution. 
15 #
16 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
17 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
20 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 import errno
28 import optparse
29 import os
30 import shutil
31 import subprocess
32 import sys
33 import zipfile
34
35 _configurationBuildDirectory = None
36 _topLevelBuildDirectory = None
37 _hostBuildDirectory = None
38
39 PATH_TO_LAUNCHER = './Tools/WebKitArchiveSupport/run-webkit-archive'
40 PATH_TO_README = './Tools/WebKitArchiveSupport/README'
41
42 def main():
43     parser = optparse.OptionParser("usage: %prog [options] [action]")
44     parser.add_option("--platform", dest="platform")
45     parser.add_option("--debug", action="store_const", const="debug", dest="configuration")
46     parser.add_option("--release", action="store_const", const="release", dest="configuration")
47     parser.add_option("--minify", action="store_true", dest="minify", default=False,
48                       help="Create a minified archive by removing files that are not necessary for running applications against the built product, at the cost of complicating debugging.")
49                       
50     options, (action, ) = parser.parse_args()
51     if not options.platform:
52         parser.error("Platform is required")
53         return 1
54     if not options.configuration:
55         parser.error("Configuration is required")
56         return 1
57     if action not in ('archive', 'extract'):
58         parser.error("Action is required")
59         return 1
60
61     genericPlatform = options.platform.split('-', 1)[0]
62     determineWebKitBuildDirectories(genericPlatform, options.platform, options.configuration)
63     if not _topLevelBuildDirectory:
64         print >> sys.stderr, 'Could not determine top-level build directory'
65         return 1
66     if not _configurationBuildDirectory:
67         print >> sys.stderr, 'Could not determine configuration-specific build directory'
68         return 1
69
70     if action == 'archive':
71         return archiveBuiltProduct(options.configuration, genericPlatform, options.platform, options.minify)
72     else:
73         return extractBuiltProduct(options.configuration, genericPlatform)
74
75
76 def webkitBuildDirectoryForConfigurationAndPlatform(configuration, platform, fullPlatform='', returnTopLevelDirectory=False):
77     if fullPlatform.startswith('ios-simulator'):
78         platform = 'ios-simulator'
79     elif platform == 'ios':
80         platform = 'device'
81     command = ['perl', os.path.join(os.path.dirname(__file__), '..', 'Scripts', 'webkit-build-directory'), '--' + platform, '--' + configuration]
82     if returnTopLevelDirectory:
83         command += ['--top-level']
84     else:
85         command += ['--configuration']
86     return subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0].strip()
87
88
89 def determineWebKitBuildDirectories(platform, fullPlatform, configuration):
90     global _configurationBuildDirectory
91     global _topLevelBuildDirectory
92     global _hostBuildDirectory
93     _configurationBuildDirectory = webkitBuildDirectoryForConfigurationAndPlatform(configuration, platform, fullPlatform)
94     _topLevelBuildDirectory = webkitBuildDirectoryForConfigurationAndPlatform(configuration, platform, fullPlatform, returnTopLevelDirectory=True)
95     if platform == 'ios':
96         _hostBuildDirectory = webkitBuildDirectoryForConfigurationAndPlatform(configuration, 'mac')
97     else:
98         _hostBuildDirectory = _configurationBuildDirectory
99     
100     return _topLevelBuildDirectory
101
102
103 def removeDirectoryIfExists(thinDirectory):
104     if os.path.isdir(thinDirectory):
105         shutil.rmtree(thinDirectory)
106
107
108 def copyBuildFiles(source, destination, patterns):
109     shutil.copytree(source, destination, ignore=shutil.ignore_patterns(*patterns))
110
111
112 def createZipFromList(listToZip, configuration, excludePattern=None):
113     archiveDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'WebKitBuild'))
114     archiveFile = os.path.join(archiveDir, configuration + '.zip')
115
116     try:
117         os.unlink(archiveFile)
118     except OSError, e:
119         if e.errno != errno.ENOENT:
120             raise
121
122     if sys.platform.startswith('linux'):
123         zipCommand = ['zip', '-y', '-r', archiveFile] + listToZip
124         if excludePattern:
125             zipCommand += ['-x', excludePattern]
126         return subprocess.call(zipCommand, cwd=_configurationBuildDirectory)
127
128     raise NotImplementedError('Unsupported platform: {platform}'.format(platform=sys.platform))
129
130
131 def createZipManually(directoryToZip, archiveFile):
132     archiveZip = zipfile.ZipFile(archiveFile, "w")
133
134     for path, dirNames, fileNames in os.walk(directoryToZip):
135         relativePath = os.path.relpath(path, directoryToZip)
136         for fileName in fileNames:
137             archiveZip.write(os.path.join(path, fileName), os.path.join(relativePath, fileName))
138
139     archiveZip.close()
140
141 def addFilesToArchive(archiveFile, pathToLauncher, pathToReadme):
142     command = ['/usr/bin/zip', '-j', archiveFile, pathToLauncher, pathToReadme]
143     return subprocess.call(command)
144
145 def createZip(directoryToZip, configuration, embedParentDirectoryNameOnDarwin=False, minify=False):
146     archiveDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "WebKitBuild"))
147     if minify:
148         configuration = 'minified-' + configuration
149     archiveFile = os.path.join(archiveDir, configuration + ".zip")
150
151     try:
152         os.unlink(archiveFile)
153     except OSError, e:
154         if e.errno != errno.ENOENT:
155             raise
156
157     if sys.platform == 'darwin':
158         command = ['ditto', '-c', '-k', '--sequesterRsrc']
159         if embedParentDirectoryNameOnDarwin:
160             command += ['--keepParent']
161         command += [directoryToZip, archiveFile]
162         return subprocess.call(command) or addFilesToArchive(archiveFile, PATH_TO_LAUNCHER, PATH_TO_README) 
163     elif sys.platform == 'cygwin':
164         return subprocess.call(["zip", "-r", archiveFile, "bin32"], cwd=directoryToZip)
165     elif sys.platform == 'win32':
166         createZipManually(directoryToZip, archiveFile)
167         return 0
168     elif sys.platform.startswith('linux'):
169         return subprocess.call(["zip", "-y", "-r", archiveFile, "."], cwd=directoryToZip)
170
171
172 def dirContainsdwo(directory):
173     sourcedir = os.path.join(_configurationBuildDirectory, directory)
174     for root, dirs, files in os.walk(sourcedir, topdown=False):
175         for name in files:
176             if name.endswith(".dwo"):
177                 return True
178     return False
179
180
181 def minifyDirectory(platform, directory):
182     if platform == 'mac' or platform == 'ios':
183         dir_patterns = ('.dSYM', 'DerivedSources')
184         file_patterns = ('.a',)
185         for root, dirs, files in os.walk(directory):
186             if any(root.endswith(pattern) for pattern in dir_patterns):
187                 print 'Removing: {}'.format(root)
188                 shutil.rmtree(root)
189             for name in files:
190                 if any(name.endswith(pattern) for pattern in file_patterns):
191                     print 'Removing: {}'.format(os.path.join(root, name))
192                     os.remove(os.path.join(root, name))
193
194
195 def archiveBuiltProduct(configuration, platform, fullPlatform, minify=False):
196     assert platform in ('mac', 'win', 'gtk', 'ios', 'wpe')
197
198     if platform == 'ios':
199         combinedDirectory = os.path.join(_topLevelBuildDirectory, 'combined-mac-and-ios')
200         removeDirectoryIfExists(combinedDirectory)
201         os.makedirs(combinedDirectory)
202         if minify:
203             minifyDirectory(platform, _configurationBuildDirectory)
204             minifyDirectory(platform, _hostBuildDirectory)
205         if subprocess.call(['/bin/cp', '-pR', _configurationBuildDirectory, combinedDirectory]):
206             return 1
207         if subprocess.call(['/bin/cp', '-pR', _hostBuildDirectory, combinedDirectory]):
208             return 1
209
210         return createZip(combinedDirectory, configuration, minify=minify)
211     elif platform == 'mac':
212         if minify:
213             minifyDirectory(platform, _configurationBuildDirectory)
214         return createZip(_configurationBuildDirectory, configuration, embedParentDirectoryNameOnDarwin=True, minify=minify)
215     elif platform == 'win':
216         # FIXME: We shouldn't hardcode the assumption of a 32-bit build. See <https://bugs.webkit.org/show_bug.cgi?id=149715>.
217         binDirectory = os.path.join(_configurationBuildDirectory, 'bin32')
218         thinDirectory = os.path.join(_configurationBuildDirectory, 'thin')
219         thinBinDirectory = os.path.join(thinDirectory, "bin32")
220
221         removeDirectoryIfExists(thinDirectory)
222         copyBuildFiles(binDirectory, thinBinDirectory, ['*.ilk'])
223         if createZip(thinDirectory, configuration):
224             return 1
225
226         shutil.rmtree(thinDirectory)
227
228     elif platform == 'gtk' or platform == 'wpe':
229         # On GTK+/WPE we don't need the intermediate step of creating a thinDirectory
230         # to be compressed in a ZIP file, because we can create the ZIP directly.
231         # This is faster and requires less disk resources.
232         neededDirectories = ['bin', 'lib']
233         # When debug fission is enabled the directories below contain dwo files
234         # with the debug information needed to generate backtraces with GDB.
235         for objectDir in ['Tools', 'Source']:
236             if dirContainsdwo(objectDir):
237                 neededDirectories.append(objectDir)
238
239         if createZipFromList(neededDirectories, configuration, excludePattern='*.o'):
240             return 1
241
242 def unzipArchive(directoryToExtractTo, configuration):
243     archiveDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "WebKitBuild"))
244     assert os.path.isdir(archiveDir)
245     archiveFile = os.path.join(archiveDir, configuration + ".zip")
246
247     if sys.platform == 'darwin':
248         if subprocess.call(["ditto", "-x", "-k", archiveFile, directoryToExtractTo]):
249             return 1
250     elif sys.platform == 'cygwin' or sys.platform.startswith('linux'):
251         if subprocess.call(["unzip", "-o", archiveFile], cwd=directoryToExtractTo):
252             return 1
253     elif sys.platform == 'win32':
254         archive = zipfile.ZipFile(archiveFile, "r")
255         archive.extractall(directoryToExtractTo)
256         archive.close()
257
258     os.unlink(archiveFile)
259
260
261 def extractBuiltProduct(configuration, platform):
262     assert platform in ('mac', 'win', 'gtk', 'ios', 'wpe')
263
264     archiveFile = os.path.join(_topLevelBuildDirectory, configuration + '.zip')
265
266     removeDirectoryIfExists(_configurationBuildDirectory)
267     os.makedirs(_configurationBuildDirectory)
268
269     if platform in ('mac', 'ios'):
270         return unzipArchive(_topLevelBuildDirectory, configuration)
271     elif platform == 'win' or platform == 'gtk' or platform == 'wpe':
272         print 'Extracting', _configurationBuildDirectory
273         return unzipArchive(_configurationBuildDirectory, configuration)
274
275
276 if __name__ == '__main__':
277     sys.exit(main())