Tools/flatpak/flatpakutils.py

   1 # Copyright (C) 2017 Igalia S.L.
   2 #
   3 # This program is free software; you can redistribute it and/or
   4 # modify it under the terms of the GNU Lesser General Public
   5 # License as published by the Free Software Foundation; either
   6 # version 2.1 of the License, or (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 # Lesser General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU Lesser General Public
  14 # License along with this program; if not, write to the
  15 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  16 # Boston, MA 02110-1301, USA.
  17 import argparse
  18 import logging
  19 try:
  20     import configparser
  21 except ImportError:
  22     import ConfigParser as configparser
  23 from contextlib import contextmanager
  24 import errno
  25 import json
  26 import multiprocessing
  27 import os
  28 import shlex
  29 import shutil
  30 import signal
  31 import subprocess
  32 import sys
  33 import tempfile
  34 import re
  35
  36 from webkitpy.common.system.systemhost import SystemHost
  37 from webkitpy.port.factory import PortFactory
  38 from webkitpy.common.system.logutils import configure_logging
  39
  40 try:
  41     from urllib.parse import urlparse  # pylint: disable=E0611
  42 except ImportError:
  43     from urlparse import urlparse
  44
  45 try:
  46     from urllib.request import urlopen  # pylint: disable=E0611
  47 except ImportError:
  48     from urllib2 import urlopen
  49
  50 FLATPAK_REQ = [
  51     ("flatpak", "0.10.0"),
  52     ("flatpak-builder", "0.10.0"),
  53 ]
  54
  55 FLATPAK_VERSION = {}
  56
  57 WPE_MANIFEST_MAP = {
  58     "qt": "org.webkit.WPEQT.yaml",
  59 }
  60
  61 scriptdir = os.path.abspath(os.path.dirname(__file__))
  62 _log = logging.getLogger(__name__)
  63
  64
  65 class Colors:
  66     HEADER = "\033[95m"
  67     OKBLUE = "\033[94m"
  68     OKGREEN = "\033[92m"
  69     WARNING = "\033[93m"
  70     FAIL = "\033[91m"
  71     ENDC = "\033[0m"
  72
  73
  74 class Console:
  75
  76     quiet = False
  77
  78     @classmethod
  79     def message(cls, str_format, *args):
  80         if cls.quiet:
  81             return
  82
  83         if args:
  84             print(str_format % args)
  85         else:
  86             print(str_format)
  87
  88         # Flush so that messages are printed at the right time
  89         # as we use many subprocesses.
  90         sys.stdout.flush()
  91
  92
  93 def check_flatpak(verbose=True):
  94     # Flatpak is only supported on Linux.
  95     if not sys.platform.startswith("linux"):
  96         return False
  97
  98     for app, required_version in FLATPAK_REQ:
  99         try:
 100             output = subprocess.check_output([app, "--version"])
 101         except (subprocess.CalledProcessError, OSError):
 102             if verbose:
 103                 Console.message("\n%sYou need to install %s >= %s"
 104                                 " to be able to use the '%s' script.\n\n"
 105                                 "You can find some informations about"
 106                                 " how to install it for your distribution at:\n"
 107                                 "    * http://flatpak.org/%s\n", Colors.FAIL,
 108                                 app, required_version, sys.argv[0], Colors.ENDC)
 109             return False
 110
 111         def comparable_version(version):
 112             return tuple(map(int, (version.split("."))))
 113
 114         version = output.decode("utf-8").split(" ")[1].strip("\n")
 115         current = comparable_version(version)
 116         FLATPAK_VERSION[app] = current
 117         if current < comparable_version(required_version):
 118             Console.message("\n%s%s %s required but %s found."
 119                             " Please update and try again%s\n", Colors.FAIL,
 120                             app, required_version, version, Colors.ENDC)
 121             return False
 122
 123     return True
 124
 125
 126 def remove_extension_points(array):
 127     result_args = []
 128     for arg in array:
 129         if(not arg.startswith('--extension')):
 130             result_args.append(arg)
 131     return result_args
 132
 133
 134 def remove_comments(string):
 135     pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
 136     # first group captures quoted strings (double or single)
 137     # second group captures comments (//single-line or /* multi-line */)
 138     regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
 139
 140     def _replacer(match):
 141         # if the 2nd group (capturing comments) is not None,
 142         # it means we have captured a non-quoted (real) comment string.
 143         if match.group(2) is not None:
 144             return ""  # so we will return empty to remove the comment
 145         else:  # otherwise, we will return the 1st group
 146             return match.group(1)  # captured quoted-string
 147     return regex.sub(_replacer, string)
 148
 149
 150 def load_manifest(manifest_path, port_name=None, command=None):
 151     is_yaml = manifest_path.endswith('.yaml')
 152     with open(manifest_path, "r") as mr:
 153         contents = mr.read()
 154
 155         contents = contents % {"COMMAND": command, "PORTNAME": port_name}
 156         if is_yaml:
 157             import yaml
 158
 159             manifest = yaml.load(contents)
 160         else:
 161             contents = remove_comments(contents)
 162             manifest = json.loads(contents)
 163
 164     return manifest
 165
 166
 167 def expand_manifest(manifest_path, outfile, port_name, source_root, command):
 168     """Creates the manifest file."""
 169     try:
 170         os.remove(outfile)
 171     except OSError:
 172         pass
 173
 174     manifest = load_manifest(manifest_path, port_name=port_name, command=command)
 175     if not manifest:
 176         return False
 177
 178     if "sdk-hash" in manifest:
 179         del manifest["sdk-hash"]
 180     if "runtime-hash" in manifest:
 181         del manifest["runtime-hash"]
 182
 183     all_modules = []
 184
 185     overriden_modules = []
 186     if "WEBKIT_EXTRA_MODULESETS" in os.environ:
 187         overriden_modules = load_manifest(os.environ["WEBKIT_EXTRA_MODULESETS"])
 188         if not overriden_modules:
 189             overriden_modules = []
 190     for modules in manifest["modules"]:
 191         submanifest_path = None
 192         if type(modules) is str:
 193             submanifest_path = os.path.join(os.path.dirname(manifest_path), modules)
 194             modules = load_manifest(submanifest_path, port_name=port_name, command=command)
 195
 196         if not isinstance(modules, list):
 197             modules = [modules]
 198
 199         for module in modules:
 200             for overriden_module in overriden_modules:
 201                 if module['name'] == overriden_module['name']:
 202                     module = overriden_module
 203                     overriden_modules.remove(module)
 204                     break
 205
 206             all_modules.append(module)
 207
 208     # And add overriden modules right before the webkit port build def.
 209     for overriden_module in overriden_modules:
 210         all_modules.insert(-1, overriden_module)
 211
 212     manifest["modules"] = all_modules
 213     for module in manifest["modules"]:
 214         if not module.get("sources"):
 215             continue
 216
 217         if module["sources"][0]["type"] == "git":
 218             if port_name == module["name"]:
 219                 repo = "file://" + source_root
 220                 module["sources"][0]["url"] = repo
 221
 222         for source in module["sources"]:
 223             if source["type"] == "patch" or (source["type"] == "file" and source.get('path')):
 224                 source["path"] = os.path.join(os.path.dirname(manifest_path), source["path"])
 225
 226     with open(outfile, "w") as of:
 227         of.write(json.dumps(manifest, indent=4))
 228
 229     return True
 230
 231
 232 class FlatpakObject:
 233
 234     def __init__(self, user):
 235         self.user = user
 236
 237     def flatpak(self, command, *args, **kwargs):
 238         show_output = kwargs.pop("show_output", False)
 239         comment = kwargs.pop("commend", None)
 240         if comment:
 241             Console.message(comment)
 242
 243         command = ["flatpak", command]
 244         if self.user:
 245             res = subprocess.check_output(command + ["--help"]).decode("utf-8")
 246             if "--user" in res:
 247                 command.append("--user")
 248         command.extend(args)
 249
 250         if not show_output:
 251             return subprocess.check_output(command).decode("utf-8")
 252
 253         return subprocess.check_call(command)
 254
 255
 256 class FlatpakPackages(FlatpakObject):
 257
 258     def __init__(self, repos, user=True):
 259         FlatpakObject.__init__(self, user=user)
 260
 261         self.repos = repos
 262
 263         self.runtimes = self.__detect_runtimes()
 264         self.apps = self.__detect_apps()
 265         self.packages = self.runtimes + self.apps
 266
 267
 268     def __detect_packages(self, *args):
 269         packs = []
 270         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
 271             out = self.flatpak("list", "-d", *args)
 272             package_defs = [line for line in out.split("\n") if line]
 273             for package_def in package_defs:
 274                 splited_packaged_def = package_def.split()
 275                 name, arch, branch = splited_packaged_def[0].split("/")
 276
 277                 # If installed from a file, the package is in no repo
 278                 repo_name = splited_packaged_def[1]
 279                 repo = self.repos.repos.get(repo_name)
 280
 281                 packs.append(FlatpakPackage(name, branch, repo, arch))
 282         else:
 283             out = self.flatpak("list", "--columns=application,arch,branch,origin", *args)
 284             package_defs = [line for line in out.split("\n") if line]
 285             for package_def in package_defs:
 286                 name, arch, branch, origin = package_def.split("\t")
 287
 288                 # If installed from a file, the package is in no repo
 289                 repo = self.repos.repos.get(origin)
 290
 291                 packs.append(FlatpakPackage(name, branch, repo, arch))
 292
 293         return packs
 294
 295
 296     def __detect_runtimes(self):
 297         return self.__detect_packages("--runtime")
 298
 299     def __detect_apps(self):
 300         return self.__detect_packages()
 301
 302     def __iter__(self):
 303         for package in self.packages:
 304             yield package
 305
 306
 307 class FlatpakRepos(FlatpakObject):
 308
 309     def __init__(self, user=True):
 310         FlatpakObject.__init__(self, user=user)
 311         self.repos = {}
 312         self.update()
 313
 314     def update(self):
 315         self.repos = {}
 316         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
 317             out = self.flatpak("remote-list", "-d")
 318             remotes = [line for line in out.split("\n") if line]
 319             for repo in remotes:
 320                 for components in [repo.split(" "), repo.split("\t")]:
 321                     if len(components) == 1:
 322                         components = repo.split("\t")
 323                     name = components[0]
 324                     desc = ""
 325                     url = None
 326                     for elem in components[1:]:
 327                         if not elem:
 328                             continue
 329                         parsed_url = urlparse(elem)
 330                         if parsed_url.scheme:
 331                             url = elem
 332                             break
 333
 334                         if desc:
 335                             desc += " "
 336                         desc += elem
 337
 338                     if url:
 339                         break
 340
 341                 if not url:
 342                     Console.message("No valid URI found for: %s", repo)
 343                     continue
 344
 345                 self.repos[name] = FlatpakRepo(name, url, desc, repos=self)
 346         else:
 347             out = self.flatpak("remote-list", "--columns=name,title,url")
 348             remotes = [line for line in out.split("\n") if line]
 349             for remote in remotes:
 350                 name, title, url = remote.split("\t")
 351                 parsed_url = urlparse(url)
 352                 if not parsed_url.scheme:
 353                     Console.message("No valid URI found for: %s", remote)
 354                     continue
 355
 356                 self.repos[name] = FlatpakRepo(name, url, title, repos=self)
 357
 358         self.packages = FlatpakPackages(self)
 359
 360     def add(self, repo, override=True):
 361         same_name = None
 362         for name, tmprepo in self.repos.items():
 363             if repo.url == tmprepo.url:
 364                 return tmprepo
 365             elif repo.name == name:
 366                 same_name = tmprepo
 367
 368         if same_name:
 369             if override:
 370                 self.flatpak("remote-modify", repo.name, "--url=" + repo.url,
 371                              comment="Setting repo %s URL from %s to %s"
 372                              % (repo.name, same_name.url, repo.url))
 373                 same_name.url = repo.url
 374
 375                 return same_name
 376             else:
 377                 return None
 378         else:
 379             self.flatpak("remote-add", repo.name, "--from", repo.repo_file.name,
 380                          "--if-not-exists",
 381                          comment="Adding repo %s" % repo.name)
 382
 383         repo.repos = self
 384         return repo
 385
 386
 387 class FlatpakRepo(FlatpakObject):
 388
 389     def __init__(self, name, desc=None, url=None,
 390                  repo_file=None, user=True, repos=None):
 391         FlatpakObject.__init__(self, user=user)
 392
 393         self.name = name
 394         self.url = url
 395         self.desc = desc
 396         self.repo_file_name = repo_file
 397         self._repo_file = None
 398         self.repos = repos
 399         assert name
 400         if repo_file and not url:
 401             repo = configparser.ConfigParser()
 402             repo.read(self.repo_file.name)
 403             self.url = repo["Flatpak Repo"]["Url"]
 404         else:
 405             assert url
 406
 407     @property
 408     def repo_file(self):
 409         if self._repo_file:
 410             return self._repo_file
 411
 412         assert self.repo_file_name
 413         self._repo_file = tempfile.NamedTemporaryFile(mode="wb")
 414         self._repo_file.write(urlopen(self.repo_file_name).read())
 415         self._repo_file.flush()
 416
 417         return self._repo_file
 418
 419
 420 class FlatpakPackage(FlatpakObject):
 421     """A flatpak app."""
 422
 423     def __init__(self, name, branch, repo, arch, user=True, hash=None):
 424         FlatpakObject.__init__(self, user=user)
 425
 426         self.name = name
 427         self.branch = str(branch)
 428         self.repo = repo
 429         self.arch = arch
 430         self.hash = hash
 431
 432     def __str__(self):
 433         return "%s/%s/%s %s" % (self.name, self.arch, self.branch, self.repo.name)
 434
 435     def is_installed(self, branch):
 436         if not self.repo:
 437             # Bundle installed from file
 438             return True
 439
 440         self.repo.repos.update()
 441         for package in self.repo.repos.packages:
 442             if package.name == self.name and \
 443                     package.branch == branch and \
 444                     package.arch == self.arch:
 445                 return True
 446
 447         return False
 448
 449     def install(self):
 450         if not self.repo:
 451             return False
 452
 453         args = ["install", self.repo.name, self.name, "--reinstall", self.branch, "--assumeyes"]
 454
 455         self.flatpak(*args, show_output=True,
 456                      comment="Installing from " + self.repo.name + " " +
 457                              self.name + " " + self.arch + " " + self.branch)
 458
 459     def update(self):
 460         if not self.is_installed(self.branch):
 461             return self.install()
 462
 463         extra_args = []
 464         comment = "Updating %s" % self.name
 465         if self.hash:
 466             extra_args = ["--commit", self.hash]
 467             comment += " to %s" % self.hash
 468
 469         extra_args.append("--assumeyes")
 470
 471         self.flatpak("update", self.name, self.branch, show_output=True,
 472                     *extra_args, comment=comment)
 473
 474
 475 @contextmanager
 476 def disable_signals(signals=[signal.SIGINT]):
 477     old_signal_handlers = []
 478
 479     for disabled_signal in signals:
 480         old_signal_handlers.append((disabled_signal, signal.getsignal(disabled_signal)))
 481         signal.signal(disabled_signal, signal.SIG_IGN)
 482
 483     yield
 484
 485     for disabled_signal, previous_handler in old_signal_handlers:
 486         signal.signal(disabled_signal, previous_handler)
 487
 488
 489 class WebkitFlatpak:
 490
 491     @staticmethod
 492     def load_from_args(args=None, add_help=True):
 493         self = WebkitFlatpak()
 494
 495         parser = argparse.ArgumentParser(prog="webkit-flatpak", add_help=add_help)
 496         general = parser.add_argument_group("General")
 497         general.add_argument('--verbose', action='store_true',
 498                              help='Show debug message')
 499         general.add_argument("--debug",
 500                             help="Compile with Debug configuration, also installs Sdk debug symboles.",
 501                             action="store_true")
 502         general.add_argument("--release", help="Compile with Release configuration.", action="store_true")
 503         general.add_argument('--platform', action='store', help='Platform to use (e.g., "mac-lion")'),
 504         general.add_argument('--gtk', action='store_const', dest='platform', const='gtk',
 505                              help='Alias for --platform=gtk')
 506         general.add_argument('--wpe', action='store_const', dest='platform', const='wpe',
 507                             help=('Alias for --platform=wpe'))
 508         general.add_argument("-nf", "--no-flatpak-update", dest="no_flatpak_update",
 509                             action="store_true",
 510                             help="Do not update flaptak runtime/sdk")
 511         general.add_argument("-u", "--update", dest="update",
 512                             action="store_true",
 513                             help="Update the runtime/sdk/app and rebuild the development environment if needed")
 514         general.add_argument("-b", "--build-webkit", dest="build_webkit",
 515                             action="store_true",
 516                             help="Force rebuilding the app.")
 517         general.add_argument("-ba", "--build-all", dest="build_all",
 518                             action="store_true",
 519                             help="Force rebuilding the app and its dependencies.")
 520         general.add_argument("-q", "--quiet", dest="quiet",
 521                             action="store_true",
 522                             help="Do not print anything")
 523         general.add_argument("-t", "--tests", dest="run_tests",
 524                             nargs=argparse.REMAINDER,
 525                             help="Run LayoutTests")
 526         general.add_argument("-c", "--command",
 527                             nargs=argparse.REMAINDER,
 528                             help="The command to run in the sandbox",
 529                             dest="user_command")
 530         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
 531         general.add_argument("--use-icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
 532         general.add_argument("--wpe-extension", action="store", dest="wpe_extension", help="WPE Extension to enable")
 533
 534         debugoptions = parser.add_argument_group("Debugging")
 535         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
 536         debugoptions.add_argument("-m", "--coredumpctl-matches", default="", help='Arguments to pass to gdb.')
 537
 538         buildoptions = parser.add_argument_group("Extra build arguments")
 539         buildoptions.add_argument("--makeargs", help="Optional Makefile flags")
 540         buildoptions.add_argument("--cmakeargs",
 541                                 help="One or more optional CMake flags (e.g. --cmakeargs=\"-DFOO=bar -DCMAKE_PREFIX_PATH=/usr/local\")")
 542
 543         general.add_argument("--clean", dest="clean", action="store_true",
 544             help="Clean previous builds and restart from scratch")
 545
 546         _, self.args = parser.parse_known_args(args=args, namespace=self)
 547
 548         return self
 549
 550     def __init__(self):
 551         self.sdk_repo = None
 552         self.runtime = None
 553         self.locale = None
 554         self.sdk = None
 555         self.sdk_debug = None
 556         self.app = None
 557
 558         self.verbose = False
 559         self.quiet = False
 560         self.packs = []
 561         self.update = False
 562         self.args = []
 563         self.finish_args = None
 564
 565         self.no_flatpak_update = False
 566         self.release = False
 567         self.debug = False
 568         self.clean = False
 569         self.run_tests = None
 570         self.source_root = os.path.normpath(os.path.abspath(os.path.join(scriptdir, '../../')))
 571         # Where the source folder is mounted inside the sandbox.
 572         self.sandbox_source_root = "/app/webkit"
 573
 574         self.build_webkit = False
 575         self.build_all = False
 576
 577         self.sdk_branch = None
 578         self.platform = "gtk"
 579         self.build_type = "Release"
 580         self.manifest_path = None
 581         self.name = None
 582         self.build_name = None
 583         self.flatpak_root_path = None
 584         self.cache_path = None
 585         self.app_module = None
 586         self.flatpak_default_args = []
 587         self.check_available = False
 588         self.wpe_extension = None
 589
 590         # Default application to run in the sandbox
 591         self.command = None
 592         self.user_command = []
 593
 594         # debug options
 595         self.gdb = None
 596         self.coredumpctl_matches = ""
 597
 598         # Extra build options
 599         self.cmakeargs = ""
 600         self.makeargs = ""
 601
 602         self.use_icecream = False
 603         self.icc_version = None
 604
 605     def clean_args(self):
 606         os.environ["FLATPAK_USER_DIR"] = os.environ.get("WEBKIT_FLATPAK_USER_DIR", os.path.realpath(os.path.join(scriptdir, "../../WebKitBuild", "UserFlatpak")))
 607         try:
 608             os.makedirs(os.environ["FLATPAK_USER_DIR"])
 609         except OSError as e:
 610             pass
 611
 612         configure_logging(logging.DEBUG if self.verbose else logging.INFO)
 613         _log.debug("Using flatpak user dir: %s" % os.environ["FLATPAK_USER_DIR"])
 614
 615         if not self.debug and not self.release:
 616             factory = PortFactory(SystemHost())
 617             port = factory.get(self.platform)
 618             self.debug = port.default_configuration() == "Debug"
 619         self.build_type = "Debug" if self.debug else "Release"
 620
 621         self.platform = self.platform.upper()
 622
 623         if self.gdb is None and '--gdb' in sys.argv:
 624             self.gdb = ""
 625
 626         self.command = "%s %s %s" % (os.path.join(self.sandbox_source_root,
 627             "Tools/Scripts/run-minibrowser"),
 628             "--" + self.platform.lower(),
 629             " --debug" if self.debug else " --release")
 630
 631         self.name = "org.webkit.%s" % self.platform
 632
 633         if self.wpe_extension:
 634             manifest_filename = WPE_MANIFEST_MAP[self.wpe_extension]
 635         else:
 636             manifest_filename = "org.webkit.WebKit.yaml"
 637         self.manifest_path = os.path.abspath(os.path.join(scriptdir, '../flatpak/') + manifest_filename)
 638
 639         self.build_name = self.name + "-generated"
 640
 641         build_root = os.path.join(self.source_root, 'WebKitBuild')
 642         self.flatpak_build_path = os.path.join(build_root, self.platform, "FlatpakTree" + self.build_type)
 643         self.cache_path = os.path.join(build_root, "FlatpakCache")
 644         self.build_path = os.path.join(build_root, self.platform, self.build_type)
 645         try:
 646             os.makedirs(self.build_path)
 647         except OSError as e:
 648             if e.errno != errno.EEXIST:
 649                 raise e
 650         self.config_file = os.path.join(self.build_path, 'webkit_flatpak_config.json')
 651
 652         Console.quiet = self.quiet
 653         if not check_flatpak():
 654             return False
 655
 656         repos = FlatpakRepos()
 657         self.sdk_repo = repos.add(
 658             FlatpakRepo("flathub",
 659                         url="https://dl.flathub.org/repo/",
 660                         repo_file="https://dl.flathub.org/repo/flathub.flatpakrepo"))
 661
 662         manifest = load_manifest(self.manifest_path, port_name=self.name)
 663         if not manifest:
 664             return False
 665
 666         self.app = manifest['app-id']
 667
 668         self.sdk_branch = manifest["runtime-version"]
 669         self.finish_args = manifest.get("finish-args", [])
 670         self.finish_args = remove_extension_points(self.finish_args)
 671         self.runtime = FlatpakPackage(manifest['runtime'], self.sdk_branch,
 672                                       self.sdk_repo, "x86_64",
 673                                       hash=manifest.get("runtime-hash"))
 674         self.locale = FlatpakPackage(manifest['runtime'] + '.Locale',
 675                                      self.sdk_branch, self.sdk_repo, "x86_64")
 676         self.sdk = FlatpakPackage(manifest['sdk'], self.sdk_branch,
 677                                   self.sdk_repo, "x86_64",
 678                                   hash=manifest.get("sdk-hash"))
 679         self.packs = [self.runtime, self.locale, self.sdk]
 680
 681         if self.debug:
 682             self.sdk_debug = FlatpakPackage(manifest['sdk'] + '.Debug', self.sdk_branch,
 683                                       self.sdk_repo, "x86_64")
 684             self.packs.append(self.sdk_debug)
 685         self.manifest_generated_path = os.path.join(self.cache_path,
 686                                                     self.build_name + ".json")
 687         try:
 688             with open(self.config_file) as config:
 689                 json_config = json.load(config)
 690                 self.icc_version = json_config['icecc_version']
 691         except IOError as e:
 692             pass
 693
 694         return True
 695
 696     def run_in_sandbox(self, *args, **kwargs):
 697         cwd = kwargs.pop("cwd", None)
 698         extra_env_vars = kwargs.pop("env", {})
 699         stdout = kwargs.pop("stdout", sys.stdout)
 700         extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
 701
 702         if not isinstance(args, list):
 703             args = list(args)
 704         if args:
 705             if os.path.exists(args[0]):
 706                 command = os.path.normpath(os.path.abspath(args[0]))
 707                 # Take into account the fact that the webkit source dir is remounted inside the sandbox.
 708                 args[0] = command.replace(self.source_root, self.sandbox_source_root)
 709             if args[0].endswith("build-webkit"):
 710                 args.append("--prefix=/app")
 711
 712         sandbox_build_path = os.path.join(self.sandbox_source_root, "WebKitBuild", self.build_type)
 713         with tempfile.NamedTemporaryFile(mode="w") as tmpscript:
 714             flatpak_command = ["flatpak", "build", "--die-with-parent",
 715                 "--bind-mount=/run/shm=/dev/shm",
 716                 # Workaround for https://webkit.org/b/187384 to have our own perl modules usable inside the sandbox
 717                 # as setting the PERL5LIB envvar won't work inside apache (and for scripts using `perl -T``).
 718                 "--bind-mount=/etc/perl=%s" % os.path.join(self.flatpak_build_path, "files/lib/perl"),
 719                 "--bind-mount=/run/host/%s=%s" % (tempfile.gettempdir(), tempfile.gettempdir()),
 720                 "--bind-mount=%s=%s" % (self.sandbox_source_root, self.source_root),
 721                 "--talk-name=org.a11y.Bus",
 722                 "--talk-name=org.gtk.vfs",
 723                 "--talk-name=org.gtk.vfs.*",
 724                 # We mount WebKitBuild/PORTNAME/BuildType to /app/webkit/WebKitBuild/BuildType
 725                 # so we can build WPE and GTK in a same source tree.
 726                 "--bind-mount=%s=%s" % (sandbox_build_path, self.build_path)]
 727
 728             forwarded = {
 729                 "WEBKIT_TOP_LEVEL": "/app/",
 730                 "TEST_RUNNER_INJECTED_BUNDLE_FILENAME": "/app/webkit/lib/libTestRunnerInjectedBundle.so",
 731                 "ICECC_VERSION": self.icc_version,
 732             }
 733
 734             env_var_prefixes_to_keep = [
 735                 "GST",
 736                 "GTK",
 737                 "G",
 738                 "JSC",
 739                 "WEBKIT",
 740                 "WEBKIT2",
 741                 "WPE",
 742                 "GIGACAGE",
 743             ]
 744
 745             env_vars_to_keep = [
 746                 "JavaScriptCoreUseJIT",
 747                 "Malloc",
 748                 "WAYLAND_DISPLAY",
 749                 "DISPLAY",
 750                 "LANG",
 751                 "NUMBER_OF_PROCESSORS",
 752                 "CCACHE_PREFIX",
 753                 "QML2_IMPORT_PATH",
 754             ]
 755
 756             if self.use_icecream:
 757                 _log.debug('Enabling the icecream compiler')
 758                 forwarded["CCACHE_PREFIX"] = "icecc"
 759                 if not os.environ.get('NUMBER_OF_PROCESSORS'):
 760                     n_cores = multiprocessing.cpu_count() * 3
 761                     _log.debug('Follow icecream recommendation for the number of cores to use: %d' % n_cores)
 762                     forwarded["NUMBER_OF_PROCESSORS"] = n_cores
 763
 764             env_vars = os.environ
 765             env_vars.update(extra_env_vars)
 766             for envvar, value in env_vars.items():
 767                 if envvar.split("_")[0] in env_var_prefixes_to_keep or envvar in env_vars_to_keep:
 768                     forwarded[envvar] = value
 769
 770             for envvar, value in forwarded.items():
 771                 flatpak_command.append("--env=%s=%s" % (envvar, value))
 772
 773             flatpak_command += self.finish_args + extra_flatpak_args + [self.flatpak_build_path]
 774
 775             shell_string = ""
 776             if args:
 777                 if cwd:
 778                     shell_string = 'cd "%s" && "%s"' % (cwd, '" "'.join(args))
 779                 else:
 780                     shell_string = '"%s"' % ('" "'.join(args))
 781             else:
 782                 shell_string = self.command
 783                 if self.args:
 784                     shell_string += ' "%s"' % '" "'.join(self.args)
 785
 786             tmpscript.write(shell_string)
 787             tmpscript.flush()
 788
 789             _log.debug('Running in sandbox: "%s" %s\n' % ('" "'.join(flatpak_command), shell_string))
 790             flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
 791
 792             try:
 793                 subprocess.check_call(flatpak_command, stdout=stdout)
 794             except subprocess.CalledProcessError as e:
 795                 sys.stderr.write(str(e) + "\n")
 796                 return e.returncode
 797             except KeyboardInterrupt:
 798                 return 0
 799
 800         return 0
 801
 802     def run(self):
 803         if self.check_available:
 804             return 0
 805
 806         if not self.clean_args():
 807             return 1
 808
 809         if self.clean:
 810             if os.path.exists(self.flatpak_build_path):
 811                 shutil.rmtree(self.flatpak_build_path)
 812             if os.path.exists(self.build_path):
 813                 shutil.rmtree(self.build_path)
 814
 815         if self.update:
 816             Console.message("Updating Flatpak environment for %s (%s)" % (
 817                 self.platform, self.build_type))
 818             if not self.no_flatpak_update:
 819                 self.update_all()
 820
 821         return self.setup_dev_env()
 822
 823     def has_environment(self):
 824         return os.path.exists(os.path.join(self.build_path, self.flatpak_build_path))
 825
 826     def save_config(self):
 827         with open(self.config_file, 'w') as config:
 828             json_config = {'icecc_version': self.icc_version}
 829             json.dump(json_config, config)
 830
 831     def setup_ccache(self):
 832         for compiler in ["c++", "cc", "clang", "clang++", "g++", "gcc"]:
 833             self.run_in_sandbox("ln", "-s", "../../usr/bin/ccache", compiler, cwd="/app/bin")
 834
 835     def setup_icecc(self):
 836         with tempfile.NamedTemporaryFile() as tmpfile:
 837             self.run_in_sandbox('icecc', '--build-native', stdout=tmpfile, cwd=self.build_path)
 838             tmpfile.flush()
 839             tmpfile.seek(0)
 840             icc_version_filename, = re.findall(r'.*creating (.*)', tmpfile.read())
 841             self.icc_version = os.path.join(self.build_path, icc_version_filename)
 842
 843     def setup_dev_env(self):
 844         if not os.path.exists(os.path.join(self.build_path, self.flatpak_build_path)) \
 845                 or self.update or self.build_all:
 846             self.install_all()
 847             Console.message("Building %s and dependencies in %s",
 848                             self.name, self.flatpak_build_path)
 849
 850             # Create environment dirs if necessary
 851             try:
 852                 os.makedirs(os.path.dirname(self.manifest_generated_path))
 853             except OSError as e:
 854                 if e.errno != errno.EEXIST:
 855                     raise e
 856             if not expand_manifest(self.manifest_path, self.manifest_generated_path,
 857                                    self.name, self.sandbox_source_root, self.command):
 858                 return 1
 859
 860             builder_args = ["flatpak-builder", "--disable-rofiles-fuse", "--state-dir",
 861                             self.cache_path, "--ccache", self.flatpak_build_path, "--force-clean",
 862                             self.manifest_generated_path]
 863             builder_args.append("--build-only")
 864             builder_args.append("--stop-at=%s" % self.app)
 865
 866             subprocess.check_call(builder_args)
 867             self.setup_ccache()
 868             self.setup_icecc()
 869             self.save_config()
 870
 871         build_type = "--debug" if self.debug else "--release"
 872         if self.build_webkit:
 873             builder = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/build-webkit'),
 874                 build_type, '--' + self.platform.lower()]
 875             if self.makeargs:
 876                 builder.append("--makeargs=%s" % self.makeargs)
 877             if self.cmakeargs:
 878                 builder.append("--cmakeargs=%s" % self.cmakeargs)
 879             Console.message("Building webkit")
 880             res = self.run_in_sandbox(*builder)
 881
 882             if res:
 883                 return res
 884         else:
 885             Console.message("Using %s prefix in %s", self.name, self.flatpak_build_path)
 886
 887         if self.run_tests is not None:
 888             test_launcher = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/run-webkit-tests'),
 889                 build_type, '--' + self.platform.lower()] + self.run_tests
 890             return self.run_in_sandbox(*test_launcher)
 891         elif self.gdb is not None:
 892             return self.run_gdb()
 893         elif self.user_command:
 894             return self.run_in_sandbox(*self.user_command)
 895         elif not self.update and not self.build_webkit:
 896             return self.run_in_sandbox()
 897
 898         return 0
 899
 900     def install_all(self):
 901         for package in self.packs:
 902             if not package.is_installed(self.sdk_branch):
 903                 package.install()
 904
 905     def run_gdb(self):
 906         with disable_signals():
 907             try:
 908                 subprocess.check_output(['which', 'coredumpctl'])
 909             except subprocess.CalledProcessError as e:
 910                 sys.stderr.write("'coredumpctl' not present on the system, can't run. (%s)\n" % e)
 911                 return e.returncode
 912
 913         # We need access to the host from the sandbox to run.
 914         with tempfile.NamedTemporaryFile() as coredump:
 915             with tempfile.NamedTemporaryFile() as stderr:
 916                 subprocess.check_call(["coredumpctl", "dump"] + shlex.split(self.coredumpctl_matches),
 917                                       stdout=coredump, stderr=stderr)
 918
 919                 with open(stderr.name, 'r') as stderrf:
 920                     stderr = stderrf.read()
 921                 executable, = re.findall(".*Executable: (.*)", stderr)
 922                 if not executable.startswith("/newroot"):
 923                     sys.stderr.write("Executable %s doesn't seem to be a flatpaked application.\n" % executable)
 924
 925                 executable = executable.replace("/newroot", "")
 926                 args = ["gdb", executable, "/run/host/%s" % coredump.name] + shlex.split(self.gdb)
 927
 928                 return self.run_in_sandbox(*args)
 929
 930     def update_all(self):
 931         for m in [self.runtime, self.sdk, self.sdk_debug]:
 932             if m:
 933                 m.update()
 934
 935
 936 def is_sandboxed():
 937     return os.path.exists("/usr/manifest.json")
 938
 939
 940 def run_in_sandbox_if_available(args):
 941     if is_sandboxed():
 942         return None
 943
 944     if not check_flatpak(verbose=False):
 945         return None
 946
 947     flatpak_runner = WebkitFlatpak.load_from_args(args, add_help=False)
 948     if not flatpak_runner.clean_args():
 949         return None
 950
 951     if not flatpak_runner.has_environment():
 952         return None
 953
 954     sys.exit(flatpak_runner.run_in_sandbox(*args))