[GTK] Generate the make dist manifest from a CMake template file
[WebKit-https.git] / Tools / gtk / make-dist.py
1 #!/usr/bin/env python
2 # Copyright (C) 2014 Igalia S.L.
3 #
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU Lesser General Public
6 # License as published by the Free Software Foundation; either
7 # version 2 of the License, or (at your option) any later version.
8 #
9 # This library is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 # Lesser General Public License for more details.
13 #
14 # You should have received a copy of the GNU Lesser General Public
15 # License along with this library; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
17
18 from __future__ import print_function
19 from contextlib import closing
20
21 import argparse
22 import errno
23 import multiprocessing
24 import os
25 import re
26 import shutil
27 import subprocess
28 import tarfile
29
30
31 def enum(**enums):
32     return type('Enum', (), enums)
33
34
35 class Rule(object):
36     Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3)
37
38     def __init__(self, type, pattern):
39         self.type = type
40         self.original_pattern = pattern
41         self.pattern = re.compile(pattern)
42
43     def test(self, file):
44         if not(self.pattern.search(file)):
45             return Rule.Result.NO_MATCH
46         return self.type
47
48
49 class Ruleset(object):
50     _global_rules = None
51
52     def __init__(self):
53         # By default, accept all files.
54         self.rules = [Rule(Rule.Result.INCLUDE, '.*')]
55
56     @classmethod
57     def global_rules(cls):
58         if not cls._global_rules:
59             cls._global_rules = Ruleset()
60         return cls._global_rules
61
62     @classmethod
63     def add_global_rule(cls, rule):
64         cls.global_rules().add_rule(rule)
65
66     def add_rule(self, rule):
67         self.rules.append(rule)
68
69     def passes(self, file):
70         allowed = False
71         for rule in self.rules:
72             result = rule.test(file)
73             if result == Rule.Result.NO_MATCH:
74                 continue
75             allowed = Rule.Result.INCLUDE == result
76         return allowed
77
78
79 class File(object):
80     def __init__(self, source_root, tarball_root):
81         self.source_root = source_root
82         self.tarball_root = tarball_root
83
84     def should_skip_file(self, path):
85         # Do not skip files explicitly added from the manifest.
86         return False
87
88     def get_files(self):
89         yield (self.source_root, self.tarball_root)
90
91
92 class Directory(object):
93     def __init__(self, source_root, tarball_root):
94         self.source_root = source_root
95         self.tarball_root = tarball_root
96         self.rules = Ruleset()
97
98         self.files_in_version_control = self.list_files_in_version_control()
99
100     def add_rule(self, rule):
101         self.rules.add_rule(rule)
102
103     def get_tarball_path(self, filename):
104         return filename.replace(self.source_root, self.tarball_root, 1)
105
106     def list_files_in_version_control(self):
107         # FIXME: Only git is supported for now.
108         p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE)
109         out = p.communicate()[0]
110         if not out:
111             return []
112         return out.rstrip('\n').split('\n')
113
114     def should_skip_file(self, path):
115         return path not in self.files_in_version_control
116
117     def get_files(self):
118         for root, dirs, files in os.walk(self.source_root):
119
120             def passes_all_rules(entry):
121                 return Ruleset.global_rules().passes(entry) and self.rules.passes(entry)
122
123             to_keep = filter(passes_all_rules, dirs)
124             del dirs[:]
125             dirs.extend(to_keep)
126
127             for file in files:
128                 file = os.path.join(root, file)
129                 if not passes_all_rules(file):
130                     continue
131                 yield (file, self.get_tarball_path(file))
132
133
134 class Manifest(object):
135     def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'):
136         self.current_directory = None
137         self.directories = []
138         self.tarball_root = tarball_root
139         self.source_root = source_root
140         self.build_root = build_root
141
142         # Normalize the tarball root so that it starts and ends with a slash.
143         if not self.tarball_root.endswith('/'):
144             self.tarball_root = self.tarball_root + '/'
145         if not self.tarball_root.startswith('/'):
146             self.tarball_root = '/' + self.tarball_root
147
148         with open(manifest_filename, 'r') as file:
149             for line in file.readlines():
150                 self.process_line(line)
151
152     def add_rule(self, rule):
153         if self.current_directory is not None:
154             self.current_directory.add_rule(rule)
155         else:
156             Ruleset.add_global_rule(rule)
157
158     def add_directory(self, directory):
159         self.current_directory = directory
160         self.directories.append(directory)
161
162     def get_full_source_path(self, source_path):
163         if not os.path.exists(source_path):
164             source_path = os.path.join(self.source_root, source_path)
165         if not os.path.exists(source_path):
166             raise Exception('Could not find directory %s' % source_path)
167         return source_path
168
169     def get_full_tarball_path(self, path):
170         return self.tarball_root + path
171
172     def get_source_and_tarball_paths_from_parts(self, parts):
173         full_source_path = self.get_full_source_path(parts[1])
174         if len(parts) > 2:
175             full_tarball_path = self.get_full_tarball_path(parts[2])
176         else:
177             full_tarball_path = self.get_full_tarball_path(parts[1])
178         return (full_source_path, full_tarball_path)
179
180     def process_line(self, line):
181         parts = line.split()
182         if not parts:
183             return
184         if parts[0].startswith("#"):
185             return
186
187         if parts[0] == "directory" and len(parts) > 1:
188             self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts)))
189         elif parts[0] == "file" and len(parts) > 1:
190             self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts)))
191         elif parts[0] == "exclude" and len(parts) > 1:
192             self.add_rule(Rule(Rule.Result.EXCLUDE, parts[1]))
193         elif parts[0] == "include" and len(parts) > 1:
194             self.add_rule(Rule(Rule.Result.INCLUDE, parts[1]))
195
196     def should_skip_file(self, directory, filename):
197         # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir.
198         if filename.startswith(self.build_root):
199             return False
200
201         return directory.should_skip_file(filename)
202
203     def get_files(self):
204         for directory in self.directories:
205             for file_tuple in directory.get_files():
206                 if self.should_skip_file(directory, file_tuple[0]):
207                     continue
208                 yield file_tuple
209
210     def create_tarfile(self, output):
211         count = 0
212         for file_tuple in self.get_files():
213             count = count + 1
214
215         with closing(tarfile.open(output, 'w')) as tarball:
216             for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1):
217                 print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r')
218                 tarball.add(file_path, tarball_path)
219         print("Wrote {0}".format(output).ljust(40))
220
221
222 class Distcheck(object):
223     BUILD_DIRECTORY_NAME = "_build"
224     INSTALL_DIRECTORY_NAME = "_install"
225
226     def __init__(self, source_root, build_root):
227         self.source_root = source_root
228         self.build_root = build_root
229
230     def extract_tarball(self, tarball_path):
231         with closing(tarfile.open(tarball_path, 'r')) as tarball:
232             tarball.extractall(self.build_root)
233
234     def configure(self, dist_dir, build_dir, install_dir):
235         def create_dir(directory, directory_type):
236             try:
237                 os.mkdir(directory)
238             except OSError, e:
239                 if e.errno != errno.EEXIST or not os.path.isdir(directory):
240                     raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e)))
241
242         create_dir(build_dir, "build")
243         create_dir(install_dir, "install")
244
245         command = ['cmake', '-DPORT=GTK', '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', dist_dir]
246         subprocess.check_call(command, cwd=build_dir)
247
248     def build(self, build_dir):
249         command = ['make']
250         make_args = os.getenv('MAKE_ARGS')
251         if make_args:
252             command.extend(make_args.split(' '))
253         else:
254             command.append('-j%d' % multiprocessing.cpu_count())
255         subprocess.check_call(command, cwd=build_dir)
256
257     def install(self, build_dir):
258         subprocess.check_call(['make', 'install'], cwd=build_dir)
259
260     def clean(self, dist_dir):
261         shutil.rmtree(dist_dir)
262
263     def check(self, tarball):
264         tarball_name, ext = os.path.splitext(os.path.basename(tarball))
265         dist_dir = os.path.join(self.build_root, tarball_name)
266         build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME)
267         install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME)
268
269         self.extract_tarball(tarball)
270         self.configure(dist_dir, build_dir, install_dir)
271         self.build(build_dir)
272         self.install(build_dir)
273         self.clean(dist_dir)
274
275 if __name__ == "__main__":
276     class FilePathAction(argparse.Action):
277         def __call__(self, parser, namespace, values, option_string=None):
278             setattr(namespace, self.dest, os.path.abspath(values))
279
280     def ensure_version_if_possible(arguments):
281         if arguments.version is not None:
282             return
283
284         pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit2/webkit2gtk-4.0.pc")
285         if os.path.isfile(pkgconfig_file):
286             p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE)
287             version = p.communicate()[0]
288             if version:
289                 arguments.version = version.rstrip('\n')
290
291
292     def get_tarball_root_and_output_filename_from_arguments(arguments):
293         tarball_root = "webkitgtk"
294         if arguments.version is not None:
295             tarball_root += '-' + arguments.version
296
297         output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar")
298         return tarball_root, output_filename
299
300     parser = argparse.ArgumentParser(description='Build a distribution bundle.')
301     parser.add_argument('-c', '--check', action='store_true',
302                         help='Check the tarball')
303     parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(),
304                         help='The top-level directory of the source distribution. ' + \
305                               'Directory for relative paths. Defaults to current directory.')
306     parser.add_argument('--version', type=str, default=None,
307                         help='The version of the tarball to generate')
308     parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(),
309                         help='The top-level path of directory of the build root. ' + \
310                               'By default is the current directory.')
311     parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.')
312
313     arguments = parser.parse_args()
314
315     # Paths in the manifest are relative to the source directory, and this script assumes that
316     # current working directory is the source directory, so change the current working directory
317     # to be the source directory.
318     os.chdir(arguments.source_dir)
319
320     ensure_version_if_possible(arguments)
321     tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments)
322
323     manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root)
324     manifest.create_tarfile(output_filename)
325
326     if arguments.check:
327         Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename)