e7af8f06de4534e82592f5593d6fc1566a9c39f4
[WebKit-https.git] / Tools / gtk / make-dist.py
1 #!/usr/bin/env python
2 # Copyright (C) 2014 Igalia S.L.
3 #
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU Lesser General Public
6 # License as published by the Free Software Foundation; either
7 # version 2 of the License, or (at your option) any later version.
8 #
9 # This library is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 # Lesser General Public License for more details.
13 #
14 # You should have received a copy of the GNU Lesser General Public
15 # License along with this library; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
17
18 from __future__ import print_function
19 from contextlib import closing
20
21 import argparse
22 import errno
23 import multiprocessing
24 import os
25 import re
26 import shutil
27 import subprocess
28 import tarfile
29
30
31 def enum(**enums):
32     return type('Enum', (), enums)
33
34
35 class Rule(object):
36     Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3)
37
38     def __init__(self, type, pattern):
39         self.type = type
40         self.original_pattern = pattern
41         self.pattern = re.compile(pattern)
42
43     def test(self, file):
44         if not(self.pattern.search(file)):
45             return Rule.Result.NO_MATCH
46         return self.type
47
48
49 class Ruleset(object):
50     _global_rules = None
51
52     def __init__(self):
53         # By default, accept all files.
54         self.rules = [Rule(Rule.Result.INCLUDE, '.*')]
55
56     @classmethod
57     def global_rules(cls):
58         if not cls._global_rules:
59             cls._global_rules = Ruleset()
60         return cls._global_rules
61
62     @classmethod
63     def add_global_rule(cls, rule):
64         cls.global_rules().add_rule(rule)
65
66     def add_rule(self, rule):
67         self.rules.append(rule)
68
69     def passes(self, file):
70         allowed = False
71         for rule in self.rules:
72             result = rule.test(file)
73             if result == Rule.Result.NO_MATCH:
74                 continue
75             allowed = Rule.Result.INCLUDE == result
76         return allowed
77
78
79 class File(object):
80     def __init__(self, source_root, tarball_root):
81         self.source_root = source_root
82         self.tarball_root = tarball_root
83
84     def should_skip_file(self, path):
85         # Do not skip files explicitly added from the manifest.
86         return False
87
88     def get_files(self):
89         yield (self.source_root, self.tarball_root)
90
91
92 class Directory(object):
93     def __init__(self, source_root, tarball_root):
94         self.source_root = source_root
95         self.tarball_root = tarball_root
96         self.rules = Ruleset()
97
98         self.files_in_version_control = self.list_files_in_version_control()
99
100     def add_rule(self, rule):
101         self.rules.add_rule(rule)
102
103     def get_tarball_path(self, filename):
104         return filename.replace(self.source_root, self.tarball_root, 1)
105
106     def list_files_in_version_control(self):
107         # FIXME: Only git is supported for now.
108         p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE)
109         out = p.communicate()[0]
110         if not out:
111             return []
112         return out.rstrip('\n').split('\n')
113
114     def should_skip_file(self, path):
115         return path not in self.files_in_version_control
116
117     def get_files(self):
118         for root, dirs, files in os.walk(self.source_root):
119
120             def passes_all_rules(entry):
121                 return Ruleset.global_rules().passes(entry) and self.rules.passes(entry)
122
123             to_keep = filter(passes_all_rules, dirs)
124             del dirs[:]
125             dirs.extend(to_keep)
126
127             for file in files:
128                 file = os.path.join(root, file)
129                 if not passes_all_rules(file):
130                     continue
131                 yield (file, self.get_tarball_path(file))
132
133
134 class Manifest(object):
135     def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'):
136         self.current_directory = None
137         self.directories = []
138         self.tarball_root = tarball_root
139         self.source_root = source_root
140         self.build_root = build_root
141
142         # Normalize the tarball root so that it starts and ends with a slash.
143         if not self.tarball_root.endswith('/'):
144             self.tarball_root = self.tarball_root + '/'
145         if not self.tarball_root.startswith('/'):
146             self.tarball_root = '/' + self.tarball_root
147
148         with open(manifest_filename, 'r') as file:
149             for line in file.readlines():
150                 self.process_line(line)
151
152     def add_rule(self, rule):
153         if self.current_directory is not None:
154             self.current_directory.add_rule(rule)
155         else:
156             Ruleset.add_global_rule(rule)
157
158     def add_directory(self, directory):
159         self.current_directory = directory
160         self.directories.append(directory)
161
162     def get_full_source_path(self, source_path):
163         if not os.path.exists(source_path):
164             source_path = os.path.join(self.source_root, source_path)
165         if not os.path.exists(source_path):
166             raise Exception('Could not find directory %s' % source_path)
167         return source_path
168
169     def get_full_tarball_path(self, path):
170         return self.tarball_root + path
171
172     def get_source_and_tarball_paths_from_parts(self, parts):
173         full_source_path = self.get_full_source_path(parts[1])
174         if len(parts) > 2:
175             full_tarball_path = self.get_full_tarball_path(parts[2])
176         else:
177             full_tarball_path = self.get_full_tarball_path(parts[1])
178         return (full_source_path, full_tarball_path)
179
180     def process_line(self, line):
181         parts = line.split()
182         if not parts:
183             return
184         if parts[0].startswith("#"):
185             return
186
187         if parts[0] == "directory" and len(parts) > 1:
188             self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts)))
189         elif parts[0] == "file" and len(parts) > 1:
190             self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts)))
191         elif parts[0] == "exclude" and len(parts) > 1:
192             self.add_rule(Rule(Rule.Result.EXCLUDE, parts[1]))
193         elif parts[0] == "include" and len(parts) > 1:
194             self.add_rule(Rule(Rule.Result.INCLUDE, parts[1]))
195         else:
196             raise Exception('Line does not begin with a correct rule:\n\t' + line)
197
198     def should_skip_file(self, directory, filename):
199         # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir.
200         if filename.startswith(self.build_root):
201             return False
202
203         return directory.should_skip_file(filename)
204
205     def get_files(self):
206         for directory in self.directories:
207             for file_tuple in directory.get_files():
208                 if self.should_skip_file(directory, file_tuple[0]):
209                     continue
210                 yield file_tuple
211
212     def create_tarfile(self, output):
213         count = 0
214         for file_tuple in self.get_files():
215             count = count + 1
216
217         with closing(tarfile.open(output, 'w')) as tarball:
218             for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1):
219                 print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r')
220                 tarball.add(file_path, tarball_path)
221         print("Wrote {0}".format(output).ljust(40))
222
223
224 class Distcheck(object):
225     BUILD_DIRECTORY_NAME = "_build"
226     INSTALL_DIRECTORY_NAME = "_install"
227
228     def __init__(self, source_root, build_root):
229         self.source_root = source_root
230         self.build_root = build_root
231
232     def extract_tarball(self, tarball_path):
233         with closing(tarfile.open(tarball_path, 'r')) as tarball:
234             tarball.extractall(self.build_root)
235
236     def configure(self, dist_dir, build_dir, install_dir, port):
237         def create_dir(directory, directory_type):
238             try:
239                 os.mkdir(directory)
240             except OSError, e:
241                 if e.errno != errno.EEXIST or not os.path.isdir(directory):
242                     raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e)))
243
244         create_dir(build_dir, "build")
245         create_dir(install_dir, "install")
246
247         command = ['cmake', '-DPORT=%s' % port, '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', dist_dir]
248         subprocess.check_call(command, cwd=build_dir)
249
250     def build(self, build_dir):
251         command = ['make']
252         make_args = os.getenv('MAKE_ARGS')
253         if make_args:
254             command.extend(make_args.split(' '))
255         else:
256             command.append('-j%d' % multiprocessing.cpu_count())
257         subprocess.check_call(command, cwd=build_dir)
258
259     def install(self, build_dir):
260         subprocess.check_call(['make', 'install'], cwd=build_dir)
261
262     def clean(self, dist_dir):
263         shutil.rmtree(dist_dir)
264
265     def check(self, tarball, port):
266         tarball_name, ext = os.path.splitext(os.path.basename(tarball))
267         dist_dir = os.path.join(self.build_root, tarball_name)
268         build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME)
269         install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME)
270
271         self.extract_tarball(tarball)
272         self.configure(dist_dir, build_dir, install_dir, port)
273         self.build(build_dir)
274         self.install(build_dir)
275         self.clean(dist_dir)
276
277 if __name__ == "__main__":
278     class FilePathAction(argparse.Action):
279         def __call__(self, parser, namespace, values, option_string=None):
280             setattr(namespace, self.dest, os.path.abspath(values))
281
282     def ensure_version_if_possible(arguments):
283         if arguments.version is not None:
284             return
285
286         pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit/webkitgtk-4.0.pc")
287         if os.path.isfile(pkgconfig_file):
288             p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE)
289             version = p.communicate()[0]
290             if version:
291                 arguments.version = version.rstrip('\n')
292
293
294     def get_tarball_root_and_output_filename_from_arguments(arguments):
295         tarball_root = arguments.tarball_name
296         if arguments.version is not None:
297             tarball_root += '-' + arguments.version
298
299         output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar")
300         return tarball_root, output_filename
301
302     parser = argparse.ArgumentParser(description='Build a distribution bundle.')
303     parser.add_argument('-c', '--check', action='store_true',
304                         help='Check the tarball')
305     parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(),
306                         help='The top-level directory of the source distribution. ' + \
307                               'Directory for relative paths. Defaults to current directory.')
308     parser.add_argument('--version', type=str, default=None,
309                         help='The version of the tarball to generate')
310     parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(),
311                         help='The top-level path of directory of the build root. ' + \
312                               'By default is the current directory.')
313     parser.add_argument('-t', '--tarball-name', type=str, default='webkitgtk',
314                         help='Base name of tarball. Defaults to "webkitgtk".')
315     parser.add_argument('-p', '--port', type=str, default='GTK',
316                         help='Port to be built in tarball check. Defaults to "GTK".')
317     parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.')
318
319     arguments = parser.parse_args()
320
321     # Paths in the manifest are relative to the source directory, and this script assumes that
322     # current working directory is the source directory, so change the current working directory
323     # to be the source directory.
324     os.chdir(arguments.source_dir)
325
326     ensure_version_if_possible(arguments)
327     tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments)
328
329     manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root)
330     manifest.create_tarfile(output_filename)
331
332     if arguments.check:
333         Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename, arguments.port)