[Flatpak] Add support for flatpak > 1.1.2
[WebKit-https.git] / Tools / flatpak / flatpakutils.py
1 # Copyright (C) 2017 Igalia S.L.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU Lesser General Public
5 # License as published by the Free Software Foundation; either
6 # version 2.1 of the License, or (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11 # Lesser General Public License for more details.
12 #
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this program; if not, write to the
15 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
16 # Boston, MA 02110-1301, USA.
17 import argparse
18 import logging
19 try:
20     import configparser
21 except ImportError:
22     import ConfigParser as configparser
23 from contextlib import contextmanager
24 import errno
25 import json
26 import multiprocessing
27 import os
28 import shlex
29 import shutil
30 import signal
31 import subprocess
32 import sys
33 import tempfile
34 import re
35
36 from webkitpy.common.system.systemhost import SystemHost
37 from webkitpy.port.factory import PortFactory
38 from webkitpy.common.system.logutils import configure_logging
39
40 try:
41     from urllib.parse import urlparse  # pylint: disable=E0611
42 except ImportError:
43     from urlparse import urlparse
44
45 try:
46     from urllib.request import urlopen  # pylint: disable=E0611
47 except ImportError:
48     from urllib2 import urlopen
49
50 FLATPAK_REQ = [
51     ("flatpak", "0.10.0"),
52     ("flatpak-builder", "0.10.0"),
53 ]
54
55 FLATPAK_VERSION = {}
56
57 WPE_MANIFEST_MAP = {
58     "qt": "org.webkit.WPEQT.yaml",
59 }
60
61 scriptdir = os.path.abspath(os.path.dirname(__file__))
62 _log = logging.getLogger(__name__)
63
64
65 class Colors:
66     HEADER = "\033[95m"
67     OKBLUE = "\033[94m"
68     OKGREEN = "\033[92m"
69     WARNING = "\033[93m"
70     FAIL = "\033[91m"
71     ENDC = "\033[0m"
72
73
74 class Console:
75
76     quiet = False
77
78     @classmethod
79     def message(cls, str_format, *args):
80         if cls.quiet:
81             return
82
83         if args:
84             print(str_format % args)
85         else:
86             print(str_format)
87
88         # Flush so that messages are printed at the right time
89         # as we use many subprocesses.
90         sys.stdout.flush()
91
92
93 def check_flatpak(verbose=True):
94     # Flatpak is only supported on Linux.
95     if not sys.platform.startswith("linux"):
96         return False
97
98     for app, required_version in FLATPAK_REQ:
99         try:
100             output = subprocess.check_output([app, "--version"])
101         except (subprocess.CalledProcessError, OSError):
102             if verbose:
103                 Console.message("\n%sYou need to install %s >= %s"
104                                 " to be able to use the '%s' script.\n\n"
105                                 "You can find some informations about"
106                                 " how to install it for your distribution at:\n"
107                                 "    * http://flatpak.org/%s\n", Colors.FAIL,
108                                 app, required_version, sys.argv[0], Colors.ENDC)
109             return False
110
111         def comparable_version(version):
112             return tuple(map(int, (version.split("."))))
113
114         version = output.decode("utf-8").split(" ")[1].strip("\n")
115         current = comparable_version(version)
116         FLATPAK_VERSION[app] = current
117         if current < comparable_version(required_version):
118             Console.message("\n%s%s %s required but %s found."
119                             " Please update and try again%s\n", Colors.FAIL,
120                             app, required_version, version, Colors.ENDC)
121             return False
122
123     return True
124
125
126 def remove_extension_points(array):
127     result_args = []
128     for arg in array:
129         if(not arg.startswith('--extension')):
130             result_args.append(arg)
131     return result_args
132
133
134 def remove_comments(string):
135     pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
136     # first group captures quoted strings (double or single)
137     # second group captures comments (//single-line or /* multi-line */)
138     regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
139
140     def _replacer(match):
141         # if the 2nd group (capturing comments) is not None,
142         # it means we have captured a non-quoted (real) comment string.
143         if match.group(2) is not None:
144             return ""  # so we will return empty to remove the comment
145         else:  # otherwise, we will return the 1st group
146             return match.group(1)  # captured quoted-string
147     return regex.sub(_replacer, string)
148
149
150 def load_manifest(manifest_path, port_name=None, command=None):
151     is_yaml = manifest_path.endswith('.yaml')
152     with open(manifest_path, "r") as mr:
153         contents = mr.read()
154
155         contents = contents % {"COMMAND": command, "PORTNAME": port_name}
156         if is_yaml:
157             import yaml
158
159             manifest = yaml.load(contents)
160         else:
161             contents = remove_comments(contents)
162             manifest = json.loads(contents)
163
164     return manifest
165
166
167 def expand_manifest(manifest_path, outfile, port_name, source_root, command):
168     """Creates the manifest file."""
169     try:
170         os.remove(outfile)
171     except OSError:
172         pass
173
174     manifest = load_manifest(manifest_path, port_name=port_name, command=command)
175     if not manifest:
176         return False
177
178     if "sdk-hash" in manifest:
179         del manifest["sdk-hash"]
180     if "runtime-hash" in manifest:
181         del manifest["runtime-hash"]
182
183     all_modules = []
184
185     overriden_modules = []
186     if "WEBKIT_EXTRA_MODULESETS" in os.environ:
187         overriden_modules = load_manifest(os.environ["WEBKIT_EXTRA_MODULESETS"])
188         if not overriden_modules:
189             overriden_modules = []
190     for modules in manifest["modules"]:
191         submanifest_path = None
192         if type(modules) is str:
193             submanifest_path = os.path.join(os.path.dirname(manifest_path), modules)
194             modules = load_manifest(submanifest_path, port_name=port_name, command=command)
195
196         if not isinstance(modules, list):
197             modules = [modules]
198
199         for module in modules:
200             for overriden_module in overriden_modules:
201                 if module['name'] == overriden_module['name']:
202                     module = overriden_module
203                     overriden_modules.remove(module)
204                     break
205
206             all_modules.append(module)
207
208     # And add overriden modules right before the webkit port build def.
209     for overriden_module in overriden_modules:
210         all_modules.insert(-1, overriden_module)
211
212     manifest["modules"] = all_modules
213     for module in manifest["modules"]:
214         if not module.get("sources"):
215             continue
216
217         if module["sources"][0]["type"] == "git":
218             if port_name == module["name"]:
219                 repo = "file://" + source_root
220                 module["sources"][0]["url"] = repo
221
222         for source in module["sources"]:
223             if source["type"] == "patch" or (source["type"] == "file" and source.get('path')):
224                 source["path"] = os.path.join(os.path.dirname(manifest_path), source["path"])
225
226     with open(outfile, "w") as of:
227         of.write(json.dumps(manifest, indent=4))
228
229     return True
230
231
232 class FlatpakObject:
233
234     def __init__(self, user):
235         self.user = user
236
237     def flatpak(self, command, *args, **kwargs):
238         show_output = kwargs.pop("show_output", False)
239         comment = kwargs.pop("commend", None)
240         if comment:
241             Console.message(comment)
242
243         command = ["flatpak", command]
244         if self.user:
245             res = subprocess.check_output(command + ["--help"]).decode("utf-8")
246             if "--user" in res:
247                 command.append("--user")
248         command.extend(args)
249
250         if not show_output:
251             return subprocess.check_output(command).decode("utf-8")
252
253         return subprocess.check_call(command)
254
255
256 class FlatpakPackages(FlatpakObject):
257
258     def __init__(self, repos, user=True):
259         FlatpakObject.__init__(self, user=user)
260
261         self.repos = repos
262
263         self.runtimes = self.__detect_runtimes()
264         self.apps = self.__detect_apps()
265         self.packages = self.runtimes + self.apps
266
267
268     def __detect_packages(self, *args):
269         packs = []
270         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
271             out = self.flatpak("list", "-d", *args)
272             package_defs = [line for line in out.split("\n") if line]
273             for package_def in package_defs:
274                 splited_packaged_def = package_def.split()
275                 name, arch, branch = splited_packaged_def[0].split("/")
276
277                 # If installed from a file, the package is in no repo
278                 repo_name = splited_packaged_def[1]
279                 repo = self.repos.repos.get(repo_name)
280
281                 packs.append(FlatpakPackage(name, branch, repo, arch))
282         else:
283             out = self.flatpak("list", "--columns=application,arch,branch,origin", *args)
284             package_defs = [line for line in out.split("\n") if line]
285             for package_def in package_defs:
286                 name, arch, branch, origin = package_def.split("\t")
287
288                 # If installed from a file, the package is in no repo
289                 repo = self.repos.repos.get(origin)
290
291                 packs.append(FlatpakPackage(name, branch, repo, arch))
292
293         return packs
294
295
296     def __detect_runtimes(self):
297         return self.__detect_packages("--runtime")
298
299     def __detect_apps(self):
300         return self.__detect_packages()
301
302     def __iter__(self):
303         for package in self.packages:
304             yield package
305
306
307 class FlatpakRepos(FlatpakObject):
308
309     def __init__(self, user=True):
310         FlatpakObject.__init__(self, user=user)
311         self.repos = {}
312         self.update()
313
314     def update(self):
315         self.repos = {}
316         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
317             out = self.flatpak("remote-list", "-d")
318             remotes = [line for line in out.split("\n") if line]
319             for repo in remotes:
320                 for components in [repo.split(" "), repo.split("\t")]:
321                     if len(components) == 1:
322                         components = repo.split("\t")
323                     name = components[0]
324                     desc = ""
325                     url = None
326                     for elem in components[1:]:
327                         if not elem:
328                             continue
329                         parsed_url = urlparse(elem)
330                         if parsed_url.scheme:
331                             url = elem
332                             break
333
334                         if desc:
335                             desc += " "
336                         desc += elem
337
338                     if url:
339                         break
340
341                 if not url:
342                     Console.message("No valid URI found for: %s", repo)
343                     continue
344
345                 self.repos[name] = FlatpakRepo(name, url, desc, repos=self)
346         else:
347             out = self.flatpak("remote-list", "--columns=name,title,url")
348             remotes = [line for line in out.split("\n") if line]
349             for remote in remotes:
350                 name, title, url = remote.split("\t")
351                 parsed_url = urlparse(url)
352                 if not parsed_url.scheme:
353                     Console.message("No valid URI found for: %s", remote)
354                     continue
355
356                 self.repos[name] = FlatpakRepo(name, url, title, repos=self)
357
358         self.packages = FlatpakPackages(self)
359
360     def add(self, repo, override=True):
361         same_name = None
362         for name, tmprepo in self.repos.items():
363             if repo.url == tmprepo.url:
364                 return tmprepo
365             elif repo.name == name:
366                 same_name = tmprepo
367
368         if same_name:
369             if override:
370                 self.flatpak("remote-modify", repo.name, "--url=" + repo.url,
371                              comment="Setting repo %s URL from %s to %s"
372                              % (repo.name, same_name.url, repo.url))
373                 same_name.url = repo.url
374
375                 return same_name
376             else:
377                 return None
378         else:
379             self.flatpak("remote-add", repo.name, "--from", repo.repo_file.name,
380                          "--if-not-exists",
381                          comment="Adding repo %s" % repo.name)
382
383         repo.repos = self
384         return repo
385
386
387 class FlatpakRepo(FlatpakObject):
388
389     def __init__(self, name, desc=None, url=None,
390                  repo_file=None, user=True, repos=None):
391         FlatpakObject.__init__(self, user=user)
392
393         self.name = name
394         self.url = url
395         self.desc = desc
396         self.repo_file_name = repo_file
397         self._repo_file = None
398         self.repos = repos
399         assert name
400         if repo_file and not url:
401             repo = configparser.ConfigParser()
402             repo.read(self.repo_file.name)
403             self.url = repo["Flatpak Repo"]["Url"]
404         else:
405             assert url
406
407     @property
408     def repo_file(self):
409         if self._repo_file:
410             return self._repo_file
411
412         assert self.repo_file_name
413         self._repo_file = tempfile.NamedTemporaryFile(mode="wb")
414         self._repo_file.write(urlopen(self.repo_file_name).read())
415         self._repo_file.flush()
416
417         return self._repo_file
418
419
420 class FlatpakPackage(FlatpakObject):
421     """A flatpak app."""
422
423     def __init__(self, name, branch, repo, arch, user=True, hash=None):
424         FlatpakObject.__init__(self, user=user)
425
426         self.name = name
427         self.branch = str(branch)
428         self.repo = repo
429         self.arch = arch
430         self.hash = hash
431
432     def __str__(self):
433         return "%s/%s/%s %s" % (self.name, self.arch, self.branch, self.repo.name)
434
435     def is_installed(self, branch):
436         if not self.repo:
437             # Bundle installed from file
438             return True
439
440         self.repo.repos.update()
441         for package in self.repo.repos.packages:
442             if package.name == self.name and \
443                     package.branch == branch and \
444                     package.arch == self.arch:
445                 return True
446
447         return False
448
449     def install(self):
450         if not self.repo:
451             return False
452
453         args = ["install", self.repo.name, self.name, "--reinstall", self.branch, "--assumeyes"]
454
455         self.flatpak(*args, show_output=True,
456                      comment="Installing from " + self.repo.name + " " +
457                              self.name + " " + self.arch + " " + self.branch)
458
459     def update(self):
460         if not self.is_installed(self.branch):
461             return self.install()
462
463         extra_args = []
464         comment = "Updating %s" % self.name
465         if self.hash:
466             extra_args = ["--commit", self.hash]
467             comment += " to %s" % self.hash
468
469         extra_args.append("--assumeyes")
470
471         self.flatpak("update", self.name, self.branch, show_output=True,
472                     *extra_args, comment=comment)
473
474
475 @contextmanager
476 def disable_signals(signals=[signal.SIGINT]):
477     old_signal_handlers = []
478
479     for disabled_signal in signals:
480         old_signal_handlers.append((disabled_signal, signal.getsignal(disabled_signal)))
481         signal.signal(disabled_signal, signal.SIG_IGN)
482
483     yield
484
485     for disabled_signal, previous_handler in old_signal_handlers:
486         signal.signal(disabled_signal, previous_handler)
487
488
489 class WebkitFlatpak:
490
491     @staticmethod
492     def load_from_args(args=None, add_help=True):
493         self = WebkitFlatpak()
494
495         parser = argparse.ArgumentParser(prog="webkit-flatpak", add_help=add_help)
496         general = parser.add_argument_group("General")
497         general.add_argument('--verbose', action='store_true',
498                              help='Show debug message')
499         general.add_argument("--debug",
500                             help="Compile with Debug configuration, also installs Sdk debug symboles.",
501                             action="store_true")
502         general.add_argument("--release", help="Compile with Release configuration.", action="store_true")
503         general.add_argument('--platform', action='store', help='Platform to use (e.g., "mac-lion")'),
504         general.add_argument('--gtk', action='store_const', dest='platform', const='gtk',
505                              help='Alias for --platform=gtk')
506         general.add_argument('--wpe', action='store_const', dest='platform', const='wpe',
507                             help=('Alias for --platform=wpe'))
508         general.add_argument("-nf", "--no-flatpak-update", dest="no_flatpak_update",
509                             action="store_true",
510                             help="Do not update flaptak runtime/sdk")
511         general.add_argument("-u", "--update", dest="update",
512                             action="store_true",
513                             help="Update the runtime/sdk/app and rebuild the development environment if needed")
514         general.add_argument("-b", "--build-webkit", dest="build_webkit",
515                             action="store_true",
516                             help="Force rebuilding the app.")
517         general.add_argument("-ba", "--build-all", dest="build_all",
518                             action="store_true",
519                             help="Force rebuilding the app and its dependencies.")
520         general.add_argument("-q", "--quiet", dest="quiet",
521                             action="store_true",
522                             help="Do not print anything")
523         general.add_argument("-t", "--tests", dest="run_tests",
524                             nargs=argparse.REMAINDER,
525                             help="Run LayoutTests")
526         general.add_argument("-c", "--command",
527                             nargs=argparse.REMAINDER,
528                             help="The command to run in the sandbox",
529                             dest="user_command")
530         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
531         general.add_argument("--use-icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
532         general.add_argument("--wpe-extension", action="store", dest="wpe_extension", help="WPE Extension to enable")
533
534         debugoptions = parser.add_argument_group("Debugging")
535         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
536         debugoptions.add_argument("-m", "--coredumpctl-matches", default="", help='Arguments to pass to gdb.')
537
538         buildoptions = parser.add_argument_group("Extra build arguments")
539         buildoptions.add_argument("--makeargs", help="Optional Makefile flags")
540         buildoptions.add_argument("--cmakeargs",
541                                 help="One or more optional CMake flags (e.g. --cmakeargs=\"-DFOO=bar -DCMAKE_PREFIX_PATH=/usr/local\")")
542
543         general.add_argument("--clean", dest="clean", action="store_true",
544             help="Clean previous builds and restart from scratch")
545
546         _, self.args = parser.parse_known_args(args=args, namespace=self)
547
548         return self
549
550     def __init__(self):
551         self.sdk_repo = None
552         self.runtime = None
553         self.locale = None
554         self.sdk = None
555         self.sdk_debug = None
556         self.app = None
557
558         self.verbose = False
559         self.quiet = False
560         self.packs = []
561         self.update = False
562         self.args = []
563         self.finish_args = None
564
565         self.no_flatpak_update = False
566         self.release = False
567         self.debug = False
568         self.clean = False
569         self.run_tests = None
570         self.source_root = os.path.normpath(os.path.abspath(os.path.join(scriptdir, '../../')))
571         # Where the source folder is mounted inside the sandbox.
572         self.sandbox_source_root = "/app/webkit"
573
574         self.build_webkit = False
575         self.build_all = False
576
577         self.sdk_branch = None
578         self.platform = "gtk"
579         self.build_type = "Release"
580         self.manifest_path = None
581         self.name = None
582         self.build_name = None
583         self.flatpak_root_path = None
584         self.cache_path = None
585         self.app_module = None
586         self.flatpak_default_args = []
587         self.check_available = False
588         self.wpe_extension = None
589
590         # Default application to run in the sandbox
591         self.command = None
592         self.user_command = []
593
594         # debug options
595         self.gdb = None
596         self.coredumpctl_matches = ""
597
598         # Extra build options
599         self.cmakeargs = ""
600         self.makeargs = ""
601
602         self.use_icecream = False
603         self.icc_version = None
604
605     def clean_args(self):
606         os.environ["FLATPAK_USER_DIR"] = os.environ.get("WEBKIT_FLATPAK_USER_DIR", os.path.realpath(os.path.join(scriptdir, "../../WebKitBuild", "UserFlatpak")))
607         try:
608             os.makedirs(os.environ["FLATPAK_USER_DIR"])
609         except OSError as e:
610             pass
611
612         configure_logging(logging.DEBUG if self.verbose else logging.INFO)
613         _log.debug("Using flatpak user dir: %s" % os.environ["FLATPAK_USER_DIR"])
614
615         if not self.debug and not self.release:
616             factory = PortFactory(SystemHost())
617             port = factory.get(self.platform)
618             self.debug = port.default_configuration() == "Debug"
619         self.build_type = "Debug" if self.debug else "Release"
620
621         self.platform = self.platform.upper()
622
623         if self.gdb is None and '--gdb' in sys.argv:
624             self.gdb = ""
625
626         self.command = "%s %s %s" % (os.path.join(self.sandbox_source_root,
627             "Tools/Scripts/run-minibrowser"),
628             "--" + self.platform.lower(),
629             " --debug" if self.debug else " --release")
630
631         self.name = "org.webkit.%s" % self.platform
632
633         if self.wpe_extension:
634             manifest_filename = WPE_MANIFEST_MAP[self.wpe_extension]
635         else:
636             manifest_filename = "org.webkit.WebKit.yaml"
637         self.manifest_path = os.path.abspath(os.path.join(scriptdir, '../flatpak/') + manifest_filename)
638
639         self.build_name = self.name + "-generated"
640
641         build_root = os.path.join(self.source_root, 'WebKitBuild')
642         self.flatpak_build_path = os.path.join(build_root, self.platform, "FlatpakTree" + self.build_type)
643         self.cache_path = os.path.join(build_root, "FlatpakCache")
644         self.build_path = os.path.join(build_root, self.platform, self.build_type)
645         try:
646             os.makedirs(self.build_path)
647         except OSError as e:
648             if e.errno != errno.EEXIST:
649                 raise e
650         self.config_file = os.path.join(self.build_path, 'webkit_flatpak_config.json')
651
652         Console.quiet = self.quiet
653         if not check_flatpak():
654             return False
655
656         repos = FlatpakRepos()
657         self.sdk_repo = repos.add(
658             FlatpakRepo("flathub",
659                         url="https://dl.flathub.org/repo/",
660                         repo_file="https://dl.flathub.org/repo/flathub.flatpakrepo"))
661
662         manifest = load_manifest(self.manifest_path, port_name=self.name)
663         if not manifest:
664             return False
665
666         self.app = manifest['app-id']
667
668         self.sdk_branch = manifest["runtime-version"]
669         self.finish_args = manifest.get("finish-args", [])
670         self.finish_args = remove_extension_points(self.finish_args)
671         self.runtime = FlatpakPackage(manifest['runtime'], self.sdk_branch,
672                                       self.sdk_repo, "x86_64",
673                                       hash=manifest.get("runtime-hash"))
674         self.locale = FlatpakPackage(manifest['runtime'] + '.Locale',
675                                      self.sdk_branch, self.sdk_repo, "x86_64")
676         self.sdk = FlatpakPackage(manifest['sdk'], self.sdk_branch,
677                                   self.sdk_repo, "x86_64",
678                                   hash=manifest.get("sdk-hash"))
679         self.packs = [self.runtime, self.locale, self.sdk]
680
681         if self.debug:
682             self.sdk_debug = FlatpakPackage(manifest['sdk'] + '.Debug', self.sdk_branch,
683                                       self.sdk_repo, "x86_64")
684             self.packs.append(self.sdk_debug)
685         self.manifest_generated_path = os.path.join(self.cache_path,
686                                                     self.build_name + ".json")
687         try:
688             with open(self.config_file) as config:
689                 json_config = json.load(config)
690                 self.icc_version = json_config['icecc_version']
691         except IOError as e:
692             pass
693
694         return True
695
696     def run_in_sandbox(self, *args, **kwargs):
697         cwd = kwargs.pop("cwd", None)
698         extra_env_vars = kwargs.pop("env", {})
699         stdout = kwargs.pop("stdout", sys.stdout)
700         extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
701
702         if not isinstance(args, list):
703             args = list(args)
704         if args:
705             if os.path.exists(args[0]):
706                 command = os.path.normpath(os.path.abspath(args[0]))
707                 # Take into account the fact that the webkit source dir is remounted inside the sandbox.
708                 args[0] = command.replace(self.source_root, self.sandbox_source_root)
709             if args[0].endswith("build-webkit"):
710                 args.append("--prefix=/app")
711
712         sandbox_build_path = os.path.join(self.sandbox_source_root, "WebKitBuild", self.build_type)
713         with tempfile.NamedTemporaryFile(mode="w") as tmpscript:
714             flatpak_command = ["flatpak", "build", "--die-with-parent",
715                 "--bind-mount=/run/shm=/dev/shm",
716                 # Workaround for https://webkit.org/b/187384 to have our own perl modules usable inside the sandbox
717                 # as setting the PERL5LIB envvar won't work inside apache (and for scripts using `perl -T``).
718                 "--bind-mount=/etc/perl=%s" % os.path.join(self.flatpak_build_path, "files/lib/perl"),
719                 "--bind-mount=/run/host/%s=%s" % (tempfile.gettempdir(), tempfile.gettempdir()),
720                 "--bind-mount=%s=%s" % (self.sandbox_source_root, self.source_root),
721                 "--talk-name=org.a11y.Bus",
722                 "--talk-name=org.gtk.vfs",
723                 "--talk-name=org.gtk.vfs.*",
724                 # We mount WebKitBuild/PORTNAME/BuildType to /app/webkit/WebKitBuild/BuildType
725                 # so we can build WPE and GTK in a same source tree.
726                 "--bind-mount=%s=%s" % (sandbox_build_path, self.build_path)]
727
728             forwarded = {
729                 "WEBKIT_TOP_LEVEL": "/app/",
730                 "TEST_RUNNER_INJECTED_BUNDLE_FILENAME": "/app/webkit/lib/libTestRunnerInjectedBundle.so",
731                 "ICECC_VERSION": self.icc_version,
732             }
733
734             env_var_prefixes_to_keep = [
735                 "GST",
736                 "GTK",
737                 "G",
738                 "JSC",
739                 "WEBKIT",
740                 "WEBKIT2",
741                 "WPE",
742                 "GIGACAGE",
743             ]
744
745             env_vars_to_keep = [
746                 "JavaScriptCoreUseJIT",
747                 "Malloc",
748                 "WAYLAND_DISPLAY",
749                 "DISPLAY",
750                 "LANG",
751                 "NUMBER_OF_PROCESSORS",
752                 "CCACHE_PREFIX",
753                 "QML2_IMPORT_PATH",
754             ]
755
756             if self.use_icecream:
757                 _log.debug('Enabling the icecream compiler')
758                 forwarded["CCACHE_PREFIX"] = "icecc"
759                 if not os.environ.get('NUMBER_OF_PROCESSORS'):
760                     n_cores = multiprocessing.cpu_count() * 3
761                     _log.debug('Follow icecream recommendation for the number of cores to use: %d' % n_cores)
762                     forwarded["NUMBER_OF_PROCESSORS"] = n_cores
763
764             env_vars = os.environ
765             env_vars.update(extra_env_vars)
766             for envvar, value in env_vars.items():
767                 if envvar.split("_")[0] in env_var_prefixes_to_keep or envvar in env_vars_to_keep:
768                     forwarded[envvar] = value
769
770             for envvar, value in forwarded.items():
771                 flatpak_command.append("--env=%s=%s" % (envvar, value))
772
773             flatpak_command += self.finish_args + extra_flatpak_args + [self.flatpak_build_path]
774
775             shell_string = ""
776             if args:
777                 if cwd:
778                     shell_string = 'cd "%s" && "%s"' % (cwd, '" "'.join(args))
779                 else:
780                     shell_string = '"%s"' % ('" "'.join(args))
781             else:
782                 shell_string = self.command
783                 if self.args:
784                     shell_string += ' "%s"' % '" "'.join(self.args)
785
786             tmpscript.write(shell_string)
787             tmpscript.flush()
788
789             _log.debug('Running in sandbox: "%s" %s\n' % ('" "'.join(flatpak_command), shell_string))
790             flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
791
792             try:
793                 subprocess.check_call(flatpak_command, stdout=stdout)
794             except subprocess.CalledProcessError as e:
795                 sys.stderr.write(str(e) + "\n")
796                 return e.returncode
797             except KeyboardInterrupt:
798                 return 0
799
800         return 0
801
802     def run(self):
803         if self.check_available:
804             return 0
805
806         if not self.clean_args():
807             return 1
808
809         if self.clean:
810             if os.path.exists(self.flatpak_build_path):
811                 shutil.rmtree(self.flatpak_build_path)
812             if os.path.exists(self.build_path):
813                 shutil.rmtree(self.build_path)
814
815         if self.update:
816             Console.message("Updating Flatpak environment for %s (%s)" % (
817                 self.platform, self.build_type))
818             if not self.no_flatpak_update:
819                 self.update_all()
820
821         return self.setup_dev_env()
822
823     def has_environment(self):
824         return os.path.exists(os.path.join(self.build_path, self.flatpak_build_path))
825
826     def save_config(self):
827         with open(self.config_file, 'w') as config:
828             json_config = {'icecc_version': self.icc_version}
829             json.dump(json_config, config)
830
831     def setup_ccache(self):
832         for compiler in ["c++", "cc", "clang", "clang++", "g++", "gcc"]:
833             self.run_in_sandbox("ln", "-s", "../../usr/bin/ccache", compiler, cwd="/app/bin")
834
835     def setup_icecc(self):
836         with tempfile.NamedTemporaryFile() as tmpfile:
837             self.run_in_sandbox('icecc', '--build-native', stdout=tmpfile, cwd=self.build_path)
838             tmpfile.flush()
839             tmpfile.seek(0)
840             icc_version_filename, = re.findall(r'.*creating (.*)', tmpfile.read())
841             self.icc_version = os.path.join(self.build_path, icc_version_filename)
842
843     def setup_dev_env(self):
844         if not os.path.exists(os.path.join(self.build_path, self.flatpak_build_path)) \
845                 or self.update or self.build_all:
846             self.install_all()
847             Console.message("Building %s and dependencies in %s",
848                             self.name, self.flatpak_build_path)
849
850             # Create environment dirs if necessary
851             try:
852                 os.makedirs(os.path.dirname(self.manifest_generated_path))
853             except OSError as e:
854                 if e.errno != errno.EEXIST:
855                     raise e
856             if not expand_manifest(self.manifest_path, self.manifest_generated_path,
857                                    self.name, self.sandbox_source_root, self.command):
858                 return 1
859
860             builder_args = ["flatpak-builder", "--disable-rofiles-fuse", "--state-dir",
861                             self.cache_path, "--ccache", self.flatpak_build_path, "--force-clean",
862                             self.manifest_generated_path]
863             builder_args.append("--build-only")
864             builder_args.append("--stop-at=%s" % self.app)
865
866             subprocess.check_call(builder_args)
867             self.setup_ccache()
868             self.setup_icecc()
869             self.save_config()
870
871         build_type = "--debug" if self.debug else "--release"
872         if self.build_webkit:
873             builder = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/build-webkit'),
874                 build_type, '--' + self.platform.lower()]
875             if self.makeargs:
876                 builder.append("--makeargs=%s" % self.makeargs)
877             if self.cmakeargs:
878                 builder.append("--cmakeargs=%s" % self.cmakeargs)
879             Console.message("Building webkit")
880             res = self.run_in_sandbox(*builder)
881
882             if res:
883                 return res
884         else:
885             Console.message("Using %s prefix in %s", self.name, self.flatpak_build_path)
886
887         if self.run_tests is not None:
888             test_launcher = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/run-webkit-tests'),
889                 build_type, '--' + self.platform.lower()] + self.run_tests
890             return self.run_in_sandbox(*test_launcher)
891         elif self.gdb is not None:
892             return self.run_gdb()
893         elif self.user_command:
894             return self.run_in_sandbox(*self.user_command)
895         elif not self.update and not self.build_webkit:
896             return self.run_in_sandbox()
897
898         return 0
899
900     def install_all(self):
901         for package in self.packs:
902             if not package.is_installed(self.sdk_branch):
903                 package.install()
904
905     def run_gdb(self):
906         with disable_signals():
907             try:
908                 subprocess.check_output(['which', 'coredumpctl'])
909             except subprocess.CalledProcessError as e:
910                 sys.stderr.write("'coredumpctl' not present on the system, can't run. (%s)\n" % e)
911                 return e.returncode
912
913         # We need access to the host from the sandbox to run.
914         with tempfile.NamedTemporaryFile() as coredump:
915             with tempfile.NamedTemporaryFile() as stderr:
916                 subprocess.check_call(["coredumpctl", "dump"] + shlex.split(self.coredumpctl_matches),
917                                       stdout=coredump, stderr=stderr)
918
919                 with open(stderr.name, 'r') as stderrf:
920                     stderr = stderrf.read()
921                 executable, = re.findall(".*Executable: (.*)", stderr)
922                 if not executable.startswith("/newroot"):
923                     sys.stderr.write("Executable %s doesn't seem to be a flatpaked application.\n" % executable)
924
925                 executable = executable.replace("/newroot", "")
926                 args = ["gdb", executable, "/run/host/%s" % coredump.name] + shlex.split(self.gdb)
927
928                 return self.run_in_sandbox(*args)
929
930     def update_all(self):
931         for m in [self.runtime, self.sdk, self.sdk_debug]:
932             if m:
933                 m.update()
934
935
936 def is_sandboxed():
937     return os.path.exists("/usr/manifest.json")
938
939
940 def run_in_sandbox_if_available(args):
941     if is_sandboxed():
942         return None
943
944     if not check_flatpak(verbose=False):
945         return None
946
947     flatpak_runner = WebkitFlatpak.load_from_args(args, add_help=False)
948     if not flatpak_runner.clean_args():
949         return None
950
951     if not flatpak_runner.has_environment():
952         return None
953
954     sys.exit(flatpak_runner.run_in_sandbox(*args))