Tools/flatpak/flatpakutils.py

   1 # Copyright (C) 2017 Igalia S.L.
   2 #
   3 # This program is free software; you can redistribute it and/or
   4 # modify it under the terms of the GNU Lesser General Public
   5 # License as published by the Free Software Foundation; either
   6 # version 2.1 of the License, or (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 # Lesser General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU Lesser General Public
  14 # License along with this program; if not, write to the
  15 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  16 # Boston, MA 02110-1301, USA.
  17 import argparse
  18 import logging
  19 try:
  20     import configparser
  21 except ImportError:
  22     import ConfigParser as configparser
  23 from contextlib import contextmanager
  24 import errno
  25 import json
  26 import multiprocessing
  27 import os
  28 import shlex
  29 import shutil
  30 import signal
  31 import subprocess
  32 import sys
  33 import tempfile
  34 import re
  35
  36 from webkitpy.common.system.systemhost import SystemHost
  37 from webkitpy.port.factory import PortFactory
  38 from webkitpy.common.system.logutils import configure_logging
  39
  40 try:
  41     from urllib.parse import urlparse  # pylint: disable=E0611
  42 except ImportError:
  43     from urlparse import urlparse
  44
  45 try:
  46     from urllib.request import urlopen  # pylint: disable=E0611
  47 except ImportError:
  48     from urllib2 import urlopen
  49
  50 FLATPAK_REQ = [
  51     ("flatpak", "0.10.0"),
  52     ("flatpak-builder", "0.10.0"),
  53 ]
  54
  55 WPE_MANIFEST_MAP = {
  56     "qt": "org.webkit.WPEQT.yaml",
  57 }
  58
  59 scriptdir = os.path.abspath(os.path.dirname(__file__))
  60 _log = logging.getLogger(__name__)
  61
  62
  63 class Colors:
  64     HEADER = "\033[95m"
  65     OKBLUE = "\033[94m"
  66     OKGREEN = "\033[92m"
  67     WARNING = "\033[93m"
  68     FAIL = "\033[91m"
  69     ENDC = "\033[0m"
  70
  71
  72 class Console:
  73
  74     quiet = False
  75
  76     @classmethod
  77     def message(cls, str_format, *args):
  78         if cls.quiet:
  79             return
  80
  81         if args:
  82             print(str_format % args)
  83         else:
  84             print(str_format)
  85
  86         # Flush so that messages are printed at the right time
  87         # as we use many subprocesses.
  88         sys.stdout.flush()
  89
  90
  91 def check_flatpak(verbose=True):
  92     # Flatpak is only supported on Linux.
  93     if not sys.platform.startswith("linux"):
  94         return False
  95
  96     for app, required_version in FLATPAK_REQ:
  97         try:
  98             output = subprocess.check_output([app, "--version"])
  99         except (subprocess.CalledProcessError, OSError):
 100             if verbose:
 101                 Console.message("\n%sYou need to install %s >= %s"
 102                                 " to be able to use the '%s' script.\n\n"
 103                                 "You can find some informations about"
 104                                 " how to install it for your distribution at:\n"
 105                                 "    * http://flatpak.org/%s\n", Colors.FAIL,
 106                                 app, required_version, sys.argv[0], Colors.ENDC)
 107             return False
 108
 109         def comparable_version(version):
 110             return tuple(map(int, (version.split("."))))
 111
 112         version = output.decode("utf-8").split(" ")[1].strip("\n")
 113         if comparable_version(version) < comparable_version(required_version):
 114             Console.message("\n%s%s %s required but %s found."
 115                             " Please update and try again%s\n", Colors.FAIL,
 116                             app, required_version, version, Colors.ENDC)
 117             return False
 118
 119     return True
 120
 121
 122 def remove_extension_points(array):
 123     result_args = []
 124     for arg in array:
 125         if(not arg.startswith('--extension')):
 126             result_args.append(arg)
 127     return result_args
 128
 129
 130 def remove_comments(string):
 131     pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
 132     # first group captures quoted strings (double or single)
 133     # second group captures comments (//single-line or /* multi-line */)
 134     regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
 135
 136     def _replacer(match):
 137         # if the 2nd group (capturing comments) is not None,
 138         # it means we have captured a non-quoted (real) comment string.
 139         if match.group(2) is not None:
 140             return ""  # so we will return empty to remove the comment
 141         else:  # otherwise, we will return the 1st group
 142             return match.group(1)  # captured quoted-string
 143     return regex.sub(_replacer, string)
 144
 145
 146 def load_manifest(manifest_path, port_name=None, command=None):
 147     is_yaml = manifest_path.endswith('.yaml')
 148     with open(manifest_path, "r") as mr:
 149         contents = mr.read()
 150
 151         contents = contents % {"COMMAND": command, "PORTNAME": port_name}
 152         if is_yaml:
 153             import yaml
 154
 155             manifest = yaml.load(contents)
 156         else:
 157             contents = remove_comments(contents)
 158             manifest = json.loads(contents)
 159
 160     return manifest
 161
 162
 163 def expand_manifest(manifest_path, outfile, port_name, source_root, command):
 164     """Creates the manifest file."""
 165     try:
 166         os.remove(outfile)
 167     except OSError:
 168         pass
 169
 170     manifest = load_manifest(manifest_path, port_name=port_name, command=command)
 171     if not manifest:
 172         return False
 173
 174     if "sdk-hash" in manifest:
 175         del manifest["sdk-hash"]
 176     if "runtime-hash" in manifest:
 177         del manifest["runtime-hash"]
 178
 179     all_modules = []
 180
 181     overriden_modules = []
 182     if "WEBKIT_EXTRA_MODULESETS" in os.environ:
 183         overriden_modules = load_manifest(os.environ["WEBKIT_EXTRA_MODULESETS"])
 184         if not overriden_modules:
 185             overriden_modules = []
 186     for modules in manifest["modules"]:
 187         submanifest_path = None
 188         if type(modules) is str:
 189             submanifest_path = os.path.join(os.path.dirname(manifest_path), modules)
 190             modules = load_manifest(submanifest_path, port_name=port_name, command=command)
 191
 192         if not isinstance(modules, list):
 193             modules = [modules]
 194
 195         for module in modules:
 196             for overriden_module in overriden_modules:
 197                 if module['name'] == overriden_module['name']:
 198                     module = overriden_module
 199                     overriden_modules.remove(module)
 200                     break
 201
 202             all_modules.append(module)
 203
 204     # And add overriden modules right before the webkit port build def.
 205     for overriden_module in overriden_modules:
 206         all_modules.insert(-1, overriden_module)
 207
 208     manifest["modules"] = all_modules
 209     for module in manifest["modules"]:
 210         if not module.get("sources"):
 211             continue
 212
 213         if module["sources"][0]["type"] == "git":
 214             if port_name == module["name"]:
 215                 repo = "file://" + source_root
 216                 module["sources"][0]["url"] = repo
 217
 218         for source in module["sources"]:
 219             if source["type"] == "patch" or (source["type"] == "file" and source.get('path')):
 220                 source["path"] = os.path.join(os.path.dirname(manifest_path), source["path"])
 221
 222     with open(outfile, "w") as of:
 223         of.write(json.dumps(manifest, indent=4))
 224
 225     return True
 226
 227
 228 class FlatpakObject:
 229
 230     def __init__(self, user):
 231         self.user = user
 232
 233     def flatpak(self, command, *args, **kwargs):
 234         show_output = kwargs.pop("show_output", False)
 235         comment = kwargs.pop("commend", None)
 236         if comment:
 237             Console.message(comment)
 238
 239         command = ["flatpak", command]
 240         if self.user:
 241             res = subprocess.check_output(command + ["--help"]).decode("utf-8")
 242             if "--user" in res:
 243                 command.append("--user")
 244         command.extend(args)
 245
 246         if not show_output:
 247             return subprocess.check_output(command).decode("utf-8")
 248
 249         return subprocess.check_call(command)
 250
 251
 252 class FlatpakPackages(FlatpakObject):
 253
 254     def __init__(self, repos, user=True):
 255         FlatpakObject.__init__(self, user=user)
 256
 257         self.repos = repos
 258
 259         self.runtimes = self.__detect_runtimes()
 260         self.apps = self.__detect_apps()
 261         self.packages = self.runtimes + self.apps
 262
 263     def __detect_packages(self, *args):
 264         packs = []
 265         package_defs = [rd
 266                         for rd in self.flatpak("list", "-d", "--all", *args).split("\n")
 267                         if rd]
 268         for package_def in package_defs:
 269             splited_packaged_def = package_def.split()
 270             name, arch, branch = splited_packaged_def[0].split("/")
 271
 272             # If installed from a file, the package is in no repo
 273             repo_name = splited_packaged_def[1]
 274             repo = self.repos.repos.get(repo_name)
 275
 276             packs.append(FlatpakPackage(name, branch, repo, arch))
 277
 278         return packs
 279
 280     def __detect_runtimes(self):
 281         return self.__detect_packages("--runtime")
 282
 283     def __detect_apps(self):
 284         return self.__detect_packages()
 285
 286     def __iter__(self):
 287         for package in self.packages:
 288             yield package
 289
 290
 291 class FlatpakRepos(FlatpakObject):
 292
 293     def __init__(self, user=True):
 294         FlatpakObject.__init__(self, user=user)
 295         self.repos = {}
 296         self.update()
 297
 298     def update(self):
 299         self.repos = {}
 300         remotes = [row
 301                    for row in self.flatpak("remote-list", "-d").split("\n")
 302                    if row]
 303         for repo in remotes:
 304             for components in [repo.split(" "), repo.split("\t")]:
 305                 if len(components) == 1:
 306                     components = repo.split("\t")
 307                 name = components[0]
 308                 desc = ""
 309                 url = None
 310                 for elem in components[1:]:
 311                     if not elem:
 312                         continue
 313                     parsed_url = urlparse(elem)
 314                     if parsed_url.scheme:
 315                         url = elem
 316                         break
 317
 318                     if desc:
 319                         desc += " "
 320                     desc += elem
 321
 322                 if url:
 323                     break
 324
 325             if not url:
 326                 Console.message("No valid URI found for: %s", repo)
 327                 continue
 328
 329             self.repos[name] = FlatpakRepo(name, url, desc, repos=self)
 330
 331         self.packages = FlatpakPackages(self)
 332
 333     def add(self, repo, override=True):
 334         same_name = None
 335         for name, tmprepo in self.repos.items():
 336             if repo.url == tmprepo.url:
 337                 return tmprepo
 338             elif repo.name == name:
 339                 same_name = tmprepo
 340
 341         if same_name:
 342             if override:
 343                 self.flatpak("remote-modify", repo.name, "--url=" + repo.url,
 344                              comment="Setting repo %s URL from %s to %s"
 345                              % (repo.name, same_name.url, repo.url))
 346                 same_name.url = repo.url
 347
 348                 return same_name
 349             else:
 350                 return None
 351         else:
 352             self.flatpak("remote-add", repo.name, "--from", repo.repo_file.name,
 353                          "--if-not-exists",
 354                          comment="Adding repo %s" % repo.name)
 355
 356         repo.repos = self
 357         return repo
 358
 359
 360 class FlatpakRepo(FlatpakObject):
 361
 362     def __init__(self, name, desc=None, url=None,
 363                  repo_file=None, user=True, repos=None):
 364         FlatpakObject.__init__(self, user=user)
 365
 366         self.name = name
 367         self.url = url
 368         self.desc = desc
 369         self.repo_file_name = repo_file
 370         self._repo_file = None
 371         self.repos = repos
 372         assert name
 373         if repo_file and not url:
 374             repo = configparser.ConfigParser()
 375             repo.read(self.repo_file.name)
 376             self.url = repo["Flatpak Repo"]["Url"]
 377         else:
 378             assert url
 379
 380     @property
 381     def repo_file(self):
 382         if self._repo_file:
 383             return self._repo_file
 384
 385         assert self.repo_file_name
 386         self._repo_file = tempfile.NamedTemporaryFile(mode="wb")
 387         self._repo_file.write(urlopen(self.repo_file_name).read())
 388         self._repo_file.flush()
 389
 390         return self._repo_file
 391
 392
 393 class FlatpakPackage(FlatpakObject):
 394     """A flatpak app."""
 395
 396     def __init__(self, name, branch, repo, arch, user=True, hash=None):
 397         FlatpakObject.__init__(self, user=user)
 398
 399         self.name = name
 400         self.branch = str(branch)
 401         self.repo = repo
 402         self.arch = arch
 403         self.hash = hash
 404
 405     def __str__(self):
 406         return "%s/%s/%s %s" % (self.name, self.arch, self.branch, self.repo.name)
 407
 408     def is_installed(self, branch):
 409         if not self.repo:
 410             # Bundle installed from file
 411             return True
 412
 413         self.repo.repos.update()
 414         for package in self.repo.repos.packages:
 415             if package.name == self.name and \
 416                     package.branch == branch and \
 417                     package.arch == self.arch:
 418                 return True
 419
 420         return False
 421
 422     def install(self):
 423         if not self.repo:
 424             return False
 425
 426         args = ["install", self.repo.name, self.name, "--reinstall", self.branch, "--assumeyes"]
 427
 428         self.flatpak(*args, show_output=True,
 429                      comment="Installing from " + self.repo.name + " " +
 430                              self.name + " " + self.arch + " " + self.branch)
 431
 432     def update(self):
 433         if not self.is_installed(self.branch):
 434             return self.install()
 435
 436         extra_args = []
 437         comment = "Updating %s" % self.name
 438         if self.hash:
 439             extra_args = ["--commit", self.hash]
 440             comment += " to %s" % self.hash
 441
 442         extra_args.append("--assumeyes")
 443
 444         self.flatpak("update", self.name, self.branch, show_output=True,
 445                     *extra_args, comment=comment)
 446
 447
 448 @contextmanager
 449 def disable_signals(signals=[signal.SIGINT]):
 450     old_signal_handlers = []
 451
 452     for disabled_signal in signals:
 453         old_signal_handlers.append((disabled_signal, signal.getsignal(disabled_signal)))
 454         signal.signal(disabled_signal, signal.SIG_IGN)
 455
 456     yield
 457
 458     for disabled_signal, previous_handler in old_signal_handlers:
 459         signal.signal(disabled_signal, previous_handler)
 460
 461
 462 class WebkitFlatpak:
 463
 464     @staticmethod
 465     def load_from_args(args=None, add_help=True):
 466         self = WebkitFlatpak()
 467
 468         parser = argparse.ArgumentParser(prog="webkit-flatpak", add_help=add_help)
 469         general = parser.add_argument_group("General")
 470         general.add_argument('--verbose', action='store_true',
 471                              help='Show debug message')
 472         general.add_argument("--debug",
 473                             help="Compile with Debug configuration, also installs Sdk debug symboles.",
 474                             action="store_true")
 475         general.add_argument("--release", help="Compile with Release configuration.", action="store_true")
 476         general.add_argument('--platform', action='store', help='Platform to use (e.g., "mac-lion")'),
 477         general.add_argument('--gtk', action='store_const', dest='platform', const='gtk',
 478                              help='Alias for --platform=gtk')
 479         general.add_argument('--wpe', action='store_const', dest='platform', const='wpe',
 480                             help=('Alias for --platform=wpe'))
 481         general.add_argument("-nf", "--no-flatpak-update", dest="no_flatpak_update",
 482                             action="store_true",
 483                             help="Do not update flaptak runtime/sdk")
 484         general.add_argument("-u", "--update", dest="update",
 485                             action="store_true",
 486                             help="Update the runtime/sdk/app and rebuild the development environment if needed")
 487         general.add_argument("-b", "--build-webkit", dest="build_webkit",
 488                             action="store_true",
 489                             help="Force rebuilding the app.")
 490         general.add_argument("-ba", "--build-all", dest="build_all",
 491                             action="store_true",
 492                             help="Force rebuilding the app and its dependencies.")
 493         general.add_argument("-q", "--quiet", dest="quiet",
 494                             action="store_true",
 495                             help="Do not print anything")
 496         general.add_argument("-t", "--tests", dest="run_tests",
 497                             nargs=argparse.REMAINDER,
 498                             help="Run LayoutTests")
 499         general.add_argument("-c", "--command",
 500                             nargs=argparse.REMAINDER,
 501                             help="The command to run in the sandbox",
 502                             dest="user_command")
 503         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
 504         general.add_argument("--use-icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
 505         general.add_argument("--wpe-extension", action="store", dest="wpe_extension", help="WPE Extension to enable")
 506
 507         debugoptions = parser.add_argument_group("Debugging")
 508         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
 509         debugoptions.add_argument("-m", "--coredumpctl-matches", default="", help='Arguments to pass to gdb.')
 510
 511         buildoptions = parser.add_argument_group("Extra build arguments")
 512         buildoptions.add_argument("--makeargs", help="Optional Makefile flags")
 513         buildoptions.add_argument("--cmakeargs",
 514                                 help="One or more optional CMake flags (e.g. --cmakeargs=\"-DFOO=bar -DCMAKE_PREFIX_PATH=/usr/local\")")
 515
 516         general.add_argument("--clean", dest="clean", action="store_true",
 517             help="Clean previous builds and restart from scratch")
 518
 519         _, self.args = parser.parse_known_args(args=args, namespace=self)
 520
 521         return self
 522
 523     def __init__(self):
 524         self.sdk_repo = None
 525         self.runtime = None
 526         self.locale = None
 527         self.sdk = None
 528         self.sdk_debug = None
 529         self.app = None
 530
 531         self.verbose = False
 532         self.quiet = False
 533         self.packs = []
 534         self.update = False
 535         self.args = []
 536         self.finish_args = None
 537
 538         self.no_flatpak_update = False
 539         self.release = False
 540         self.debug = False
 541         self.clean = False
 542         self.run_tests = None
 543         self.source_root = os.path.normpath(os.path.abspath(os.path.join(scriptdir, '../../')))
 544         # Where the source folder is mounted inside the sandbox.
 545         self.sandbox_source_root = "/app/webkit"
 546
 547         self.build_webkit = False
 548         self.build_all = False
 549
 550         self.sdk_branch = None
 551         self.platform = "gtk"
 552         self.build_type = "Release"
 553         self.manifest_path = None
 554         self.name = None
 555         self.build_name = None
 556         self.flatpak_root_path = None
 557         self.cache_path = None
 558         self.app_module = None
 559         self.flatpak_default_args = []
 560         self.check_available = False
 561         self.wpe_extension = None
 562
 563         # Default application to run in the sandbox
 564         self.command = None
 565         self.user_command = []
 566
 567         # debug options
 568         self.gdb = None
 569         self.coredumpctl_matches = ""
 570
 571         # Extra build options
 572         self.cmakeargs = ""
 573         self.makeargs = ""
 574
 575         self.use_icecream = False
 576         self.icc_version = None
 577
 578     def clean_args(self):
 579         os.environ["FLATPAK_USER_DIR"] = os.environ.get("WEBKIT_FLATPAK_USER_DIR", os.path.realpath(os.path.join(scriptdir, "../../WebKitBuild", "UserFlatpak")))
 580         try:
 581             os.makedirs(os.environ["FLATPAK_USER_DIR"])
 582         except OSError as e:
 583             pass
 584
 585         configure_logging(logging.DEBUG if self.verbose else logging.INFO)
 586         _log.debug("Using flatpak user dir: %s" % os.environ["FLATPAK_USER_DIR"])
 587
 588         if not self.debug and not self.release:
 589             factory = PortFactory(SystemHost())
 590             port = factory.get(self.platform)
 591             self.debug = port.default_configuration() == "Debug"
 592         self.build_type = "Debug" if self.debug else "Release"
 593
 594         self.platform = self.platform.upper()
 595
 596         if self.gdb is None and '--gdb' in sys.argv:
 597             self.gdb = ""
 598
 599         self.command = "%s %s %s" % (os.path.join(self.sandbox_source_root,
 600             "Tools/Scripts/run-minibrowser"),
 601             "--" + self.platform.lower(),
 602             " --debug" if self.debug else " --release")
 603
 604         self.name = "org.webkit.%s" % self.platform
 605
 606         if self.wpe_extension:
 607             manifest_filename = WPE_MANIFEST_MAP[self.wpe_extension]
 608         else:
 609             manifest_filename = "org.webkit.WebKit.yaml"
 610         self.manifest_path = os.path.abspath(os.path.join(scriptdir, '../flatpak/') + manifest_filename)
 611
 612         self.build_name = self.name + "-generated"
 613
 614         build_root = os.path.join(self.source_root, 'WebKitBuild')
 615         self.flatpak_build_path = os.path.join(build_root, self.platform, "FlatpakTree" + self.build_type)
 616         self.cache_path = os.path.join(build_root, "FlatpakCache")
 617         self.build_path = os.path.join(build_root, self.platform, self.build_type)
 618         try:
 619             os.makedirs(self.build_path)
 620         except OSError as e:
 621             if e.errno != errno.EEXIST:
 622                 raise e
 623         self.config_file = os.path.join(self.build_path, 'webkit_flatpak_config.json')
 624
 625         Console.quiet = self.quiet
 626         if not check_flatpak():
 627             return False
 628
 629         repos = FlatpakRepos()
 630         self.sdk_repo = repos.add(
 631             FlatpakRepo("flathub",
 632                         url="https://dl.flathub.org/repo/",
 633                         repo_file="https://dl.flathub.org/repo/flathub.flatpakrepo"))
 634
 635         manifest = load_manifest(self.manifest_path, port_name=self.name)
 636         if not manifest:
 637             return False
 638
 639         self.app = manifest['app-id']
 640
 641         self.sdk_branch = manifest["runtime-version"]
 642         self.finish_args = manifest.get("finish-args", [])
 643         self.finish_args = remove_extension_points(self.finish_args)
 644         self.runtime = FlatpakPackage(manifest['runtime'], self.sdk_branch,
 645                                       self.sdk_repo, "x86_64",
 646                                       hash=manifest.get("runtime-hash"))
 647         self.locale = FlatpakPackage(manifest['runtime'] + '.Locale',
 648                                      self.sdk_branch, self.sdk_repo, "x86_64")
 649         self.sdk = FlatpakPackage(manifest['sdk'], self.sdk_branch,
 650                                   self.sdk_repo, "x86_64",
 651                                   hash=manifest.get("sdk-hash"))
 652         self.packs = [self.runtime, self.locale, self.sdk]
 653
 654         if self.debug:
 655             self.sdk_debug = FlatpakPackage(manifest['sdk'] + '.Debug', self.sdk_branch,
 656                                       self.sdk_repo, "x86_64")
 657             self.packs.append(self.sdk_debug)
 658         self.manifest_generated_path = os.path.join(self.cache_path,
 659                                                     self.build_name + ".json")
 660         try:
 661             with open(self.config_file) as config:
 662                 json_config = json.load(config)
 663                 self.icc_version = json_config['icecc_version']
 664         except IOError as e:
 665             pass
 666
 667         return True
 668
 669     def run_in_sandbox(self, *args, **kwargs):
 670         cwd = kwargs.pop("cwd", None)
 671         extra_env_vars = kwargs.pop("env", {})
 672         stdout = kwargs.pop("stdout", sys.stdout)
 673         extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
 674
 675         if not isinstance(args, list):
 676             args = list(args)
 677         if args:
 678             if os.path.exists(args[0]):
 679                 command = os.path.normpath(os.path.abspath(args[0]))
 680                 # Take into account the fact that the webkit source dir is remounted inside the sandbox.
 681                 args[0] = command.replace(self.source_root, self.sandbox_source_root)
 682             if args[0].endswith("build-webkit"):
 683                 args.append("--prefix=/app")
 684
 685         sandbox_build_path = os.path.join(self.sandbox_source_root, "WebKitBuild", self.build_type)
 686         with tempfile.NamedTemporaryFile(mode="w") as tmpscript:
 687             flatpak_command = ["flatpak", "build", "--die-with-parent",
 688                 "--bind-mount=/run/shm=/dev/shm",
 689                 # Workaround for https://webkit.org/b/187384 to have our own perl modules usable inside the sandbox
 690                 # as setting the PERL5LIB envvar won't work inside apache (and for scripts using `perl -T``).
 691                 "--bind-mount=/etc/perl=%s" % os.path.join(self.flatpak_build_path, "files/lib/perl"),
 692                 "--bind-mount=/run/host/%s=%s" % (tempfile.gettempdir(), tempfile.gettempdir()),
 693                 "--bind-mount=%s=%s" % (self.sandbox_source_root, self.source_root),
 694                 "--talk-name=org.a11y.Bus",
 695                 "--talk-name=org.gtk.vfs",
 696                 "--talk-name=org.gtk.vfs.*",
 697                 # We mount WebKitBuild/PORTNAME/BuildType to /app/webkit/WebKitBuild/BuildType
 698                 # so we can build WPE and GTK in a same source tree.
 699                 "--bind-mount=%s=%s" % (sandbox_build_path, self.build_path)]
 700
 701             forwarded = {
 702                 "WEBKIT_TOP_LEVEL": "/app/",
 703                 "TEST_RUNNER_INJECTED_BUNDLE_FILENAME": "/app/webkit/lib/libTestRunnerInjectedBundle.so",
 704                 "ICECC_VERSION": self.icc_version,
 705             }
 706
 707             env_var_prefixes_to_keep = [
 708                 "GST",
 709                 "GTK",
 710                 "G",
 711                 "JSC",
 712                 "WEBKIT",
 713                 "WEBKIT2",
 714                 "WPE",
 715                 "GIGACAGE",
 716             ]
 717
 718             env_vars_to_keep = [
 719                 "JavaScriptCoreUseJIT",
 720                 "Malloc",
 721                 "WAYLAND_DISPLAY",
 722                 "DISPLAY",
 723                 "LANG",
 724                 "NUMBER_OF_PROCESSORS",
 725                 "CCACHE_PREFIX",
 726                 "QML2_IMPORT_PATH",
 727             ]
 728
 729             if self.use_icecream:
 730                 _log.debug('Enabling the icecream compiler')
 731                 forwarded["CCACHE_PREFIX"] = "icecc"
 732                 if not os.environ.get('NUMBER_OF_PROCESSORS'):
 733                     n_cores = multiprocessing.cpu_count() * 3
 734                     _log.debug('Follow icecream recommendation for the number of cores to use: %d' % n_cores)
 735                     forwarded["NUMBER_OF_PROCESSORS"] = n_cores
 736
 737             env_vars = os.environ
 738             env_vars.update(extra_env_vars)
 739             for envvar, value in env_vars.items():
 740                 if envvar.split("_")[0] in env_var_prefixes_to_keep or envvar in env_vars_to_keep:
 741                     forwarded[envvar] = value
 742
 743             for envvar, value in forwarded.items():
 744                 flatpak_command.append("--env=%s=%s" % (envvar, value))
 745
 746             flatpak_command += self.finish_args + extra_flatpak_args + [self.flatpak_build_path]
 747
 748             shell_string = ""
 749             if args:
 750                 if cwd:
 751                     shell_string = 'cd "%s" && "%s"' % (cwd, '" "'.join(args))
 752                 else:
 753                     shell_string = '"%s"' % ('" "'.join(args))
 754             else:
 755                 shell_string = self.command
 756                 if self.args:
 757                     shell_string += ' "%s"' % '" "'.join(self.args)
 758
 759             tmpscript.write(shell_string)
 760             tmpscript.flush()
 761
 762             _log.debug('Running in sandbox: "%s" %s\n' % ('" "'.join(flatpak_command), shell_string))
 763             flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
 764
 765             try:
 766                 subprocess.check_call(flatpak_command, stdout=stdout)
 767             except subprocess.CalledProcessError as e:
 768                 sys.stderr.write(str(e) + "\n")
 769                 return e.returncode
 770             except KeyboardInterrupt:
 771                 return 0
 772
 773         return 0
 774
 775     def run(self):
 776         if self.check_available:
 777             return 0
 778
 779         if not self.clean_args():
 780             return 1
 781
 782         if self.clean:
 783             if os.path.exists(self.flatpak_build_path):
 784                 shutil.rmtree(self.flatpak_build_path)
 785             if os.path.exists(self.build_path):
 786                 shutil.rmtree(self.build_path)
 787
 788         if self.update:
 789             Console.message("Updating Flatpak environment for %s (%s)" % (
 790                 self.platform, self.build_type))
 791             if not self.no_flatpak_update:
 792                 self.update_all()
 793
 794         return self.setup_dev_env()
 795
 796     def has_environment(self):
 797         return os.path.exists(os.path.join(self.build_path, self.flatpak_build_path))
 798
 799     def save_config(self):
 800         with open(self.config_file, 'w') as config:
 801             json_config = {'icecc_version': self.icc_version}
 802             json.dump(json_config, config)
 803
 804     def setup_ccache(self):
 805         for compiler in ["c++", "cc", "clang", "clang++", "g++", "gcc"]:
 806             self.run_in_sandbox("ln", "-s", "../../usr/bin/ccache", compiler, cwd="/app/bin")
 807
 808     def setup_icecc(self):
 809         with tempfile.NamedTemporaryFile() as tmpfile:
 810             self.run_in_sandbox('icecc', '--build-native', stdout=tmpfile, cwd=self.build_path)
 811             tmpfile.flush()
 812             tmpfile.seek(0)
 813             icc_version_filename, = re.findall(r'.*creating (.*)', tmpfile.read())
 814             self.icc_version = os.path.join(self.build_path, icc_version_filename)
 815
 816     def setup_dev_env(self):
 817         if not os.path.exists(os.path.join(self.build_path, self.flatpak_build_path)) \
 818                 or self.update or self.build_all:
 819             self.install_all()
 820             Console.message("Building %s and dependencies in %s",
 821                             self.name, self.flatpak_build_path)
 822
 823             # Create environment dirs if necessary
 824             try:
 825                 os.makedirs(os.path.dirname(self.manifest_generated_path))
 826             except OSError as e:
 827                 if e.errno != errno.EEXIST:
 828                     raise e
 829             if not expand_manifest(self.manifest_path, self.manifest_generated_path,
 830                                    self.name, self.sandbox_source_root, self.command):
 831                 return 1
 832
 833             builder_args = ["flatpak-builder", "--disable-rofiles-fuse", "--state-dir",
 834                             self.cache_path, "--ccache", self.flatpak_build_path, "--force-clean",
 835                             self.manifest_generated_path]
 836             builder_args.append("--build-only")
 837             builder_args.append("--stop-at=%s" % self.app)
 838
 839             subprocess.check_call(builder_args)
 840             self.setup_ccache()
 841             self.setup_icecc()
 842             self.save_config()
 843
 844         build_type = "--debug" if self.debug else "--release"
 845         if self.build_webkit:
 846             builder = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/build-webkit'),
 847                 build_type, '--' + self.platform.lower()]
 848             if self.makeargs:
 849                 builder.append("--makeargs=%s" % self.makeargs)
 850             if self.cmakeargs:
 851                 builder.append("--cmakeargs=%s" % self.cmakeargs)
 852             Console.message("Building webkit")
 853             res = self.run_in_sandbox(*builder)
 854
 855             if res:
 856                 return res
 857         else:
 858             Console.message("Using %s prefix in %s", self.name, self.flatpak_build_path)
 859
 860         if self.run_tests is not None:
 861             test_launcher = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/run-webkit-tests'),
 862                 build_type, '--' + self.platform.lower()] + self.run_tests
 863             return self.run_in_sandbox(*test_launcher)
 864         elif self.gdb is not None:
 865             return self.run_gdb()
 866         elif self.user_command:
 867             return self.run_in_sandbox(*self.user_command)
 868         elif not self.update and not self.build_webkit:
 869             return self.run_in_sandbox()
 870
 871         return 0
 872
 873     def install_all(self):
 874         for package in self.packs:
 875             if not package.is_installed(self.sdk_branch):
 876                 package.install()
 877
 878     def run_gdb(self):
 879         with disable_signals():
 880             try:
 881                 subprocess.check_output(['which', 'coredumpctl'])
 882             except subprocess.CalledProcessError as e:
 883                 sys.stderr.write("'coredumpctl' not present on the system, can't run. (%s)\n" % e)
 884                 return e.returncode
 885
 886         # We need access to the host from the sandbox to run.
 887         with tempfile.NamedTemporaryFile() as coredump:
 888             with tempfile.NamedTemporaryFile() as stderr:
 889                 subprocess.check_call(["coredumpctl", "dump"] + shlex.split(self.coredumpctl_matches),
 890                                       stdout=coredump, stderr=stderr)
 891
 892                 with open(stderr.name, 'r') as stderrf:
 893                     stderr = stderrf.read()
 894                 executable, = re.findall(".*Executable: (.*)", stderr)
 895                 if not executable.startswith("/newroot"):
 896                     sys.stderr.write("Executable %s doesn't seem to be a flatpaked application.\n" % executable)
 897
 898                 executable = executable.replace("/newroot", "")
 899                 args = ["gdb", executable, "/run/host/%s" % coredump.name] + shlex.split(self.gdb)
 900
 901                 return self.run_in_sandbox(*args)
 902
 903     def update_all(self):
 904         for m in [self.runtime, self.sdk, self.sdk_debug]:
 905             if m:
 906                 m.update()
 907
 908
 909 def is_sandboxed():
 910     return os.path.exists("/usr/manifest.json")
 911
 912
 913 def run_in_sandbox_if_available(args):
 914     if is_sandboxed():
 915         return None
 916
 917     if not check_flatpak(verbose=False):
 918         return None
 919
 920     flatpak_runner = WebkitFlatpak.load_from_args(args, add_help=False)
 921     if not flatpak_runner.clean_args():
 922         return None
 923
 924     if not flatpak_runner.has_environment():
 925         return None
 926
 927     sys.exit(flatpak_runner.run_in_sandbox(*args))