Tools/gtk/make-dist.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2014 Igalia S.L.
   3 #
   4 # This library is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU Lesser General Public
   6 # License as published by the Free Software Foundation; either
   7 # version 2 of the License, or (at your option) any later version.
   8 #
   9 # This library is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # Lesser General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Lesser General Public
  15 # License along with this library; if not, write to the Free Software
  16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 from __future__ import print_function
  19 from contextlib import closing
  20
  21 import argparse
  22 import errno
  23 import multiprocessing
  24 import os
  25 import re
  26 import shutil
  27 import subprocess
  28 import tarfile
  29
  30
  31 def enum(**enums):
  32     return type('Enum', (), enums)
  33
  34
  35 class Rule(object):
  36     Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3)
  37
  38     def __init__(self, type, pattern):
  39         self.type = type
  40         self.original_pattern = pattern
  41         self.pattern = re.compile(pattern)
  42
  43     def test(self, file):
  44         if not(self.pattern.search(file)):
  45             return Rule.Result.NO_MATCH
  46         return self.type
  47
  48
  49 class Ruleset(object):
  50     _global_rules = None
  51
  52     def __init__(self):
  53         # By default, accept all files.
  54         self.rules = [Rule(Rule.Result.INCLUDE, '.*')]
  55
  56     @classmethod
  57     def global_rules(cls):
  58         if not cls._global_rules:
  59             cls._global_rules = Ruleset()
  60         return cls._global_rules
  61
  62     @classmethod
  63     def add_global_rule(cls, rule):
  64         cls.global_rules().add_rule(rule)
  65
  66     def add_rule(self, rule):
  67         self.rules.append(rule)
  68
  69     def passes(self, file):
  70         allowed = False
  71         for rule in self.rules:
  72             result = rule.test(file)
  73             if result == Rule.Result.NO_MATCH:
  74                 continue
  75             allowed = Rule.Result.INCLUDE == result
  76         return allowed
  77
  78
  79 class File(object):
  80     def __init__(self, source_root, tarball_root):
  81         self.source_root = source_root
  82         self.tarball_root = tarball_root
  83
  84     def should_skip_file(self, path):
  85         # Do not skip files explicitly added from the manifest.
  86         return False
  87
  88     def get_files(self):
  89         yield (self.source_root, self.tarball_root)
  90
  91
  92 class Directory(object):
  93     def __init__(self, source_root, tarball_root):
  94         self.source_root = source_root
  95         self.tarball_root = tarball_root
  96         self.rules = Ruleset()
  97
  98         self.files_in_version_control = self.list_files_in_version_control()
  99
 100     def add_rule(self, rule):
 101         self.rules.add_rule(rule)
 102
 103     def get_tarball_path(self, filename):
 104         return filename.replace(self.source_root, self.tarball_root, 1)
 105
 106     def list_files_in_version_control(self):
 107         # FIXME: Only git is supported for now.
 108         p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE)
 109         out = p.communicate()[0]
 110         if not out:
 111             return []
 112         return out.rstrip('\n').split('\n')
 113
 114     def should_skip_file(self, path):
 115         return path not in self.files_in_version_control
 116
 117     def get_files(self):
 118         for root, dirs, files in os.walk(self.source_root):
 119
 120             def passes_all_rules(entry):
 121                 return Ruleset.global_rules().passes(entry) and self.rules.passes(entry)
 122
 123             to_keep = filter(passes_all_rules, dirs)
 124             del dirs[:]
 125             dirs.extend(to_keep)
 126
 127             for file in files:
 128                 file = os.path.join(root, file)
 129                 if not passes_all_rules(file):
 130                     continue
 131                 yield (file, self.get_tarball_path(file))
 132
 133
 134 class Manifest(object):
 135     def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'):
 136         self.current_directory = None
 137         self.directories = []
 138         self.tarball_root = tarball_root
 139         self.source_root = source_root
 140         self.build_root = build_root
 141
 142         # Normalize the tarball root so that it starts and ends with a slash.
 143         if not self.tarball_root.endswith('/'):
 144             self.tarball_root = self.tarball_root + '/'
 145         if not self.tarball_root.startswith('/'):
 146             self.tarball_root = '/' + self.tarball_root
 147
 148         with open(manifest_filename, 'r') as file:
 149             for line in file.readlines():
 150                 self.process_line(line)
 151
 152     def add_rule(self, rule):
 153         if self.current_directory is not None:
 154             self.current_directory.add_rule(rule)
 155         else:
 156             Ruleset.add_global_rule(rule)
 157
 158     def add_directory(self, directory):
 159         self.current_directory = directory
 160         self.directories.append(directory)
 161
 162     def get_full_source_path(self, source_path):
 163         if not os.path.exists(source_path):
 164             source_path = os.path.join(self.source_root, source_path)
 165         if not os.path.exists(source_path):
 166             raise Exception('Could not find directory %s' % source_path)
 167         return source_path
 168
 169     def get_full_tarball_path(self, path):
 170         return self.tarball_root + path
 171
 172     def get_source_and_tarball_paths_from_parts(self, parts):
 173         full_source_path = self.get_full_source_path(parts[1])
 174         if len(parts) > 2:
 175             full_tarball_path = self.get_full_tarball_path(parts[2])
 176         else:
 177             full_tarball_path = self.get_full_tarball_path(parts[1])
 178         return (full_source_path, full_tarball_path)
 179
 180     def process_line(self, line):
 181         parts = line.split()
 182         if not parts:
 183             return
 184         if parts[0].startswith("#"):
 185             return
 186
 187         if parts[0] == "directory" and len(parts) > 1:
 188             self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts)))
 189         elif parts[0] == "file" and len(parts) > 1:
 190             self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts)))
 191         elif parts[0] == "exclude" and len(parts) > 1:
 192             self.add_rule(Rule(Rule.Result.EXCLUDE, parts[1]))
 193         elif parts[0] == "include" and len(parts) > 1:
 194             self.add_rule(Rule(Rule.Result.INCLUDE, parts[1]))
 195         else:
 196             raise Exception('Line does not begin with a correct rule:\n\t' + line)
 197
 198     def should_skip_file(self, directory, filename):
 199         # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir.
 200         if filename.startswith(self.build_root):
 201             return False
 202
 203         return directory.should_skip_file(filename)
 204
 205     def get_files(self):
 206         for directory in self.directories:
 207             for file_tuple in directory.get_files():
 208                 if self.should_skip_file(directory, file_tuple[0]):
 209                     continue
 210                 yield file_tuple
 211
 212     def create_tarfile(self, output):
 213         count = 0
 214         for file_tuple in self.get_files():
 215             count = count + 1
 216
 217         with closing(tarfile.open(output, 'w')) as tarball:
 218             for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1):
 219                 print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r')
 220                 tarball.add(file_path, tarball_path)
 221         print("Wrote {0}".format(output).ljust(40))
 222
 223
 224 class Distcheck(object):
 225     BUILD_DIRECTORY_NAME = "_build"
 226     INSTALL_DIRECTORY_NAME = "_install"
 227
 228     def __init__(self, source_root, build_root):
 229         self.source_root = source_root
 230         self.build_root = build_root
 231
 232     def extract_tarball(self, tarball_path):
 233         with closing(tarfile.open(tarball_path, 'r')) as tarball:
 234             tarball.extractall(self.build_root)
 235
 236     def configure(self, dist_dir, build_dir, install_dir, port):
 237         def create_dir(directory, directory_type):
 238             try:
 239                 os.mkdir(directory)
 240             except OSError, e:
 241                 if e.errno != errno.EEXIST or not os.path.isdir(directory):
 242                     raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e)))
 243
 244         create_dir(build_dir, "build")
 245         create_dir(install_dir, "install")
 246
 247         command = ['cmake', '-DPORT=%s' % port, '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', dist_dir]
 248         subprocess.check_call(command, cwd=build_dir)
 249
 250     def build(self, build_dir):
 251         command = ['make']
 252         make_args = os.getenv('MAKE_ARGS')
 253         if make_args:
 254             command.extend(make_args.split(' '))
 255         else:
 256             command.append('-j%d' % multiprocessing.cpu_count())
 257         subprocess.check_call(command, cwd=build_dir)
 258
 259     def install(self, build_dir):
 260         subprocess.check_call(['make', 'install'], cwd=build_dir)
 261
 262     def clean(self, dist_dir):
 263         shutil.rmtree(dist_dir)
 264
 265     def check(self, tarball, port):
 266         tarball_name, ext = os.path.splitext(os.path.basename(tarball))
 267         dist_dir = os.path.join(self.build_root, tarball_name)
 268         build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME)
 269         install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME)
 270
 271         self.extract_tarball(tarball)
 272         self.configure(dist_dir, build_dir, install_dir, port)
 273         self.build(build_dir)
 274         self.install(build_dir)
 275         self.clean(dist_dir)
 276
 277 if __name__ == "__main__":
 278     class FilePathAction(argparse.Action):
 279         def __call__(self, parser, namespace, values, option_string=None):
 280             setattr(namespace, self.dest, os.path.abspath(values))
 281
 282     def ensure_version_if_possible(arguments):
 283         if arguments.version is not None:
 284             return
 285
 286         pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit/webkitgtk-4.0.pc")
 287         if os.path.isfile(pkgconfig_file):
 288             p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE)
 289             version = p.communicate()[0]
 290             if version:
 291                 arguments.version = version.rstrip('\n')
 292
 293
 294     def get_tarball_root_and_output_filename_from_arguments(arguments):
 295         tarball_root = arguments.tarball_name
 296         if arguments.version is not None:
 297             tarball_root += '-' + arguments.version
 298
 299         output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar")
 300         return tarball_root, output_filename
 301
 302     parser = argparse.ArgumentParser(description='Build a distribution bundle.')
 303     parser.add_argument('-c', '--check', action='store_true',
 304                         help='Check the tarball')
 305     parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(),
 306                         help='The top-level directory of the source distribution. ' + \
 307                               'Directory for relative paths. Defaults to current directory.')
 308     parser.add_argument('--version', type=str, default=None,
 309                         help='The version of the tarball to generate')
 310     parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(),
 311                         help='The top-level path of directory of the build root. ' + \
 312                               'By default is the current directory.')
 313     parser.add_argument('-t', '--tarball-name', type=str, default='webkitgtk',
 314                         help='Base name of tarball. Defaults to "webkitgtk".')
 315     parser.add_argument('-p', '--port', type=str, default='GTK',
 316                         help='Port to be built in tarball check. Defaults to "GTK".')
 317     parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.')
 318
 319     arguments = parser.parse_args()
 320
 321     # Paths in the manifest are relative to the source directory, and this script assumes that
 322     # current working directory is the source directory, so change the current working directory
 323     # to be the source directory.
 324     os.chdir(arguments.source_dir)
 325
 326     ensure_version_if_possible(arguments)
 327     tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments)
 328
 329     manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root)
 330     manifest.create_tarfile(output_filename)
 331
 332     if arguments.check:
 333         Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename, arguments.port)