5484b48c56e1866d7484ead07230a35cf86d6228
[WebKit-https.git] / Tools / flatpak / flatpakutils.py
1 # Copyright (C) 2017 Igalia S.L.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU Lesser General Public
5 # License as published by the Free Software Foundation; either
6 # version 2.1 of the License, or (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11 # Lesser General Public License for more details.
12 #
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this program; if not, write to the
15 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
16 # Boston, MA 02110-1301, USA.
17 import argparse
18 import logging
19 try:
20     import configparser
21 except ImportError:
22     import ConfigParser as configparser
23 from contextlib import contextmanager
24 import errno
25 import json
26 import multiprocessing
27 import os
28 import shlex
29 import shutil
30 import signal
31 import subprocess
32 import sys
33 import tempfile
34 import re
35
36 from webkitpy.common.system.systemhost import SystemHost
37 from webkitpy.port.factory import PortFactory
38 from webkitpy.common.system.logutils import configure_logging
39
40 try:
41     from urllib.parse import urlparse  # pylint: disable=E0611
42 except ImportError:
43     from urlparse import urlparse
44
45 try:
46     from urllib.request import urlopen  # pylint: disable=E0611
47 except ImportError:
48     from urllib2 import urlopen
49
50 FLATPAK_REQ = [
51     ("flatpak", "0.10.0"),
52     ("flatpak-builder", "0.10.0"),
53 ]
54
55 WPE_MANIFEST_MAP = {
56     "qt": "org.webkit.WPEQT.yaml",
57 }
58
59 scriptdir = os.path.abspath(os.path.dirname(__file__))
60 _log = logging.getLogger(__name__)
61
62
63 class Colors:
64     HEADER = "\033[95m"
65     OKBLUE = "\033[94m"
66     OKGREEN = "\033[92m"
67     WARNING = "\033[93m"
68     FAIL = "\033[91m"
69     ENDC = "\033[0m"
70
71
72 class Console:
73
74     quiet = False
75
76     @classmethod
77     def message(cls, str_format, *args):
78         if cls.quiet:
79             return
80
81         if args:
82             print(str_format % args)
83         else:
84             print(str_format)
85
86         # Flush so that messages are printed at the right time
87         # as we use many subprocesses.
88         sys.stdout.flush()
89
90
91 def check_flatpak(verbose=True):
92     # Flatpak is only supported on Linux.
93     if not sys.platform.startswith("linux"):
94         return False
95
96     for app, required_version in FLATPAK_REQ:
97         try:
98             output = subprocess.check_output([app, "--version"])
99         except (subprocess.CalledProcessError, OSError):
100             if verbose:
101                 Console.message("\n%sYou need to install %s >= %s"
102                                 " to be able to use the '%s' script.\n\n"
103                                 "You can find some informations about"
104                                 " how to install it for your distribution at:\n"
105                                 "    * http://flatpak.org/%s\n", Colors.FAIL,
106                                 app, required_version, sys.argv[0], Colors.ENDC)
107             return False
108
109         def comparable_version(version):
110             return tuple(map(int, (version.split("."))))
111
112         version = output.decode("utf-8").split(" ")[1].strip("\n")
113         if comparable_version(version) < comparable_version(required_version):
114             Console.message("\n%s%s %s required but %s found."
115                             " Please update and try again%s\n", Colors.FAIL,
116                             app, required_version, version, Colors.ENDC)
117             return False
118
119     return True
120
121
122 def remove_extension_points(array):
123     result_args = []
124     for arg in array:
125         if(not arg.startswith('--extension')):
126             result_args.append(arg)
127     return result_args
128
129
130 def remove_comments(string):
131     pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
132     # first group captures quoted strings (double or single)
133     # second group captures comments (//single-line or /* multi-line */)
134     regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
135
136     def _replacer(match):
137         # if the 2nd group (capturing comments) is not None,
138         # it means we have captured a non-quoted (real) comment string.
139         if match.group(2) is not None:
140             return ""  # so we will return empty to remove the comment
141         else:  # otherwise, we will return the 1st group
142             return match.group(1)  # captured quoted-string
143     return regex.sub(_replacer, string)
144
145
146 def load_manifest(manifest_path, port_name=None, command=None):
147     is_yaml = manifest_path.endswith('.yaml')
148     with open(manifest_path, "r") as mr:
149         contents = mr.read()
150
151         contents = contents % {"COMMAND": command, "PORTNAME": port_name}
152         if is_yaml:
153             import yaml
154
155             manifest = yaml.load(contents)
156         else:
157             contents = remove_comments(contents)
158             manifest = json.loads(contents)
159
160     return manifest
161
162
163 def expand_manifest(manifest_path, outfile, port_name, source_root, command):
164     """Creates the manifest file."""
165     try:
166         os.remove(outfile)
167     except OSError:
168         pass
169
170     manifest = load_manifest(manifest_path, port_name=port_name, command=command)
171     if not manifest:
172         return False
173
174     if "sdk-hash" in manifest:
175         del manifest["sdk-hash"]
176     if "runtime-hash" in manifest:
177         del manifest["runtime-hash"]
178
179     all_modules = []
180
181     overriden_modules = []
182     if "WEBKIT_EXTRA_MODULESETS" in os.environ:
183         overriden_modules = load_manifest(os.environ["WEBKIT_EXTRA_MODULESETS"])
184         if not overriden_modules:
185             overriden_modules = []
186     for modules in manifest["modules"]:
187         submanifest_path = None
188         if type(modules) is str:
189             submanifest_path = os.path.join(os.path.dirname(manifest_path), modules)
190             modules = load_manifest(submanifest_path, port_name=port_name, command=command)
191
192         if not isinstance(modules, list):
193             modules = [modules]
194
195         for module in modules:
196             for overriden_module in overriden_modules:
197                 if module['name'] == overriden_module['name']:
198                     module = overriden_module
199                     overriden_modules.remove(module)
200                     break
201
202             all_modules.append(module)
203
204     # And add overriden modules right before the webkit port build def.
205     for overriden_module in overriden_modules:
206         all_modules.insert(-1, overriden_module)
207
208     manifest["modules"] = all_modules
209     for module in manifest["modules"]:
210         if not module.get("sources"):
211             continue
212
213         if module["sources"][0]["type"] == "git":
214             if port_name == module["name"]:
215                 repo = "file://" + source_root
216                 module["sources"][0]["url"] = repo
217
218         for source in module["sources"]:
219             if source["type"] == "patch" or (source["type"] == "file" and source.get('path')):
220                 source["path"] = os.path.join(os.path.dirname(manifest_path), source["path"])
221
222     with open(outfile, "w") as of:
223         of.write(json.dumps(manifest, indent=4))
224
225     return True
226
227
228 class FlatpakObject:
229
230     def __init__(self, user):
231         self.user = user
232
233     def flatpak(self, command, *args, **kwargs):
234         show_output = kwargs.pop("show_output", False)
235         comment = kwargs.pop("commend", None)
236         if comment:
237             Console.message(comment)
238
239         command = ["flatpak", command]
240         if self.user:
241             res = subprocess.check_output(command + ["--help"]).decode("utf-8")
242             if "--user" in res:
243                 command.append("--user")
244         command.extend(args)
245
246         if not show_output:
247             return subprocess.check_output(command).decode("utf-8")
248
249         return subprocess.check_call(command)
250
251
252 class FlatpakPackages(FlatpakObject):
253
254     def __init__(self, repos, user=True):
255         FlatpakObject.__init__(self, user=user)
256
257         self.repos = repos
258
259         self.runtimes = self.__detect_runtimes()
260         self.apps = self.__detect_apps()
261         self.packages = self.runtimes + self.apps
262
263     def __detect_packages(self, *args):
264         packs = []
265         package_defs = [rd
266                         for rd in self.flatpak("list", "-d", "--all", *args).split("\n")
267                         if rd]
268         for package_def in package_defs:
269             splited_packaged_def = package_def.split()
270             name, arch, branch = splited_packaged_def[0].split("/")
271
272             # If installed from a file, the package is in no repo
273             repo_name = splited_packaged_def[1]
274             repo = self.repos.repos.get(repo_name)
275
276             packs.append(FlatpakPackage(name, branch, repo, arch))
277
278         return packs
279
280     def __detect_runtimes(self):
281         return self.__detect_packages("--runtime")
282
283     def __detect_apps(self):
284         return self.__detect_packages()
285
286     def __iter__(self):
287         for package in self.packages:
288             yield package
289
290
291 class FlatpakRepos(FlatpakObject):
292
293     def __init__(self, user=True):
294         FlatpakObject.__init__(self, user=user)
295         self.repos = {}
296         self.update()
297
298     def update(self):
299         self.repos = {}
300         remotes = [row
301                    for row in self.flatpak("remote-list", "-d").split("\n")
302                    if row]
303         for repo in remotes:
304             for components in [repo.split(" "), repo.split("\t")]:
305                 if len(components) == 1:
306                     components = repo.split("\t")
307                 name = components[0]
308                 desc = ""
309                 url = None
310                 for elem in components[1:]:
311                     if not elem:
312                         continue
313                     parsed_url = urlparse(elem)
314                     if parsed_url.scheme:
315                         url = elem
316                         break
317
318                     if desc:
319                         desc += " "
320                     desc += elem
321
322                 if url:
323                     break
324
325             if not url:
326                 Console.message("No valid URI found for: %s", repo)
327                 continue
328
329             self.repos[name] = FlatpakRepo(name, url, desc, repos=self)
330
331         self.packages = FlatpakPackages(self)
332
333     def add(self, repo, override=True):
334         same_name = None
335         for name, tmprepo in self.repos.items():
336             if repo.url == tmprepo.url:
337                 return tmprepo
338             elif repo.name == name:
339                 same_name = tmprepo
340
341         if same_name:
342             if override:
343                 self.flatpak("remote-modify", repo.name, "--url=" + repo.url,
344                              comment="Setting repo %s URL from %s to %s"
345                              % (repo.name, same_name.url, repo.url))
346                 same_name.url = repo.url
347
348                 return same_name
349             else:
350                 return None
351         else:
352             self.flatpak("remote-add", repo.name, "--from", repo.repo_file.name,
353                          "--if-not-exists",
354                          comment="Adding repo %s" % repo.name)
355
356         repo.repos = self
357         return repo
358
359
360 class FlatpakRepo(FlatpakObject):
361
362     def __init__(self, name, desc=None, url=None,
363                  repo_file=None, user=True, repos=None):
364         FlatpakObject.__init__(self, user=user)
365
366         self.name = name
367         self.url = url
368         self.desc = desc
369         self.repo_file_name = repo_file
370         self._repo_file = None
371         self.repos = repos
372         assert name
373         if repo_file and not url:
374             repo = configparser.ConfigParser()
375             repo.read(self.repo_file.name)
376             self.url = repo["Flatpak Repo"]["Url"]
377         else:
378             assert url
379
380     @property
381     def repo_file(self):
382         if self._repo_file:
383             return self._repo_file
384
385         assert self.repo_file_name
386         self._repo_file = tempfile.NamedTemporaryFile(mode="wb")
387         self._repo_file.write(urlopen(self.repo_file_name).read())
388         self._repo_file.flush()
389
390         return self._repo_file
391
392
393 class FlatpakPackage(FlatpakObject):
394     """A flatpak app."""
395
396     def __init__(self, name, branch, repo, arch, user=True, hash=None):
397         FlatpakObject.__init__(self, user=user)
398
399         self.name = name
400         self.branch = str(branch)
401         self.repo = repo
402         self.arch = arch
403         self.hash = hash
404
405     def __str__(self):
406         return "%s/%s/%s %s" % (self.name, self.arch, self.branch, self.repo.name)
407
408     def is_installed(self, branch):
409         if not self.repo:
410             # Bundle installed from file
411             return True
412
413         self.repo.repos.update()
414         for package in self.repo.repos.packages:
415             if package.name == self.name and \
416                     package.branch == branch and \
417                     package.arch == self.arch:
418                 return True
419
420         return False
421
422     def install(self):
423         if not self.repo:
424             return False
425
426         args = ["install", self.repo.name, self.name, "--reinstall", self.branch, "--assumeyes"]
427
428         self.flatpak(*args, show_output=True,
429                      comment="Installing from " + self.repo.name + " " +
430                              self.name + " " + self.arch + " " + self.branch)
431
432     def update(self):
433         if not self.is_installed(self.branch):
434             return self.install()
435
436         extra_args = []
437         comment = "Updating %s" % self.name
438         if self.hash:
439             extra_args = ["--commit", self.hash]
440             comment += " to %s" % self.hash
441
442         extra_args.append("--assumeyes")
443
444         self.flatpak("update", self.name, self.branch, show_output=True,
445                     *extra_args, comment=comment)
446
447
448 @contextmanager
449 def disable_signals(signals=[signal.SIGINT]):
450     old_signal_handlers = []
451
452     for disabled_signal in signals:
453         old_signal_handlers.append((disabled_signal, signal.getsignal(disabled_signal)))
454         signal.signal(disabled_signal, signal.SIG_IGN)
455
456     yield
457
458     for disabled_signal, previous_handler in old_signal_handlers:
459         signal.signal(disabled_signal, previous_handler)
460
461
462 class WebkitFlatpak:
463
464     @staticmethod
465     def load_from_args(args=None, add_help=True):
466         self = WebkitFlatpak()
467
468         parser = argparse.ArgumentParser(prog="webkit-flatpak", add_help=add_help)
469         general = parser.add_argument_group("General")
470         general.add_argument('--verbose', action='store_true',
471                              help='Show debug message')
472         general.add_argument("--debug",
473                             help="Compile with Debug configuration, also installs Sdk debug symboles.",
474                             action="store_true")
475         general.add_argument("--release", help="Compile with Release configuration.", action="store_true")
476         general.add_argument('--platform', action='store', help='Platform to use (e.g., "mac-lion")'),
477         general.add_argument('--gtk', action='store_const', dest='platform', const='gtk',
478                              help='Alias for --platform=gtk')
479         general.add_argument('--wpe', action='store_const', dest='platform', const='wpe',
480                             help=('Alias for --platform=wpe'))
481         general.add_argument("-nf", "--no-flatpak-update", dest="no_flatpak_update",
482                             action="store_true",
483                             help="Do not update flaptak runtime/sdk")
484         general.add_argument("-u", "--update", dest="update",
485                             action="store_true",
486                             help="Update the runtime/sdk/app and rebuild the development environment if needed")
487         general.add_argument("-b", "--build-webkit", dest="build_webkit",
488                             action="store_true",
489                             help="Force rebuilding the app.")
490         general.add_argument("-ba", "--build-all", dest="build_all",
491                             action="store_true",
492                             help="Force rebuilding the app and its dependencies.")
493         general.add_argument("-q", "--quiet", dest="quiet",
494                             action="store_true",
495                             help="Do not print anything")
496         general.add_argument("-t", "--tests", dest="run_tests",
497                             nargs=argparse.REMAINDER,
498                             help="Run LayoutTests")
499         general.add_argument("-c", "--command",
500                             nargs=argparse.REMAINDER,
501                             help="The command to run in the sandbox",
502                             dest="user_command")
503         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
504         general.add_argument("--use-icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
505         general.add_argument("--wpe-extension", action="store", dest="wpe_extension", help="WPE Extension to enable")
506
507         debugoptions = parser.add_argument_group("Debugging")
508         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
509         debugoptions.add_argument("-m", "--coredumpctl-matches", default="", help='Arguments to pass to gdb.')
510
511         buildoptions = parser.add_argument_group("Extra build arguments")
512         buildoptions.add_argument("--makeargs", help="Optional Makefile flags")
513         buildoptions.add_argument("--cmakeargs",
514                                 help="One or more optional CMake flags (e.g. --cmakeargs=\"-DFOO=bar -DCMAKE_PREFIX_PATH=/usr/local\")")
515
516         general.add_argument("--clean", dest="clean", action="store_true",
517             help="Clean previous builds and restart from scratch")
518
519         _, self.args = parser.parse_known_args(args=args, namespace=self)
520
521         return self
522
523     def __init__(self):
524         self.sdk_repo = None
525         self.runtime = None
526         self.locale = None
527         self.sdk = None
528         self.sdk_debug = None
529         self.app = None
530
531         self.verbose = False
532         self.quiet = False
533         self.packs = []
534         self.update = False
535         self.args = []
536         self.finish_args = None
537
538         self.no_flatpak_update = False
539         self.release = False
540         self.debug = False
541         self.clean = False
542         self.run_tests = None
543         self.source_root = os.path.normpath(os.path.abspath(os.path.join(scriptdir, '../../')))
544         # Where the source folder is mounted inside the sandbox.
545         self.sandbox_source_root = "/app/webkit"
546
547         self.build_webkit = False
548         self.build_all = False
549
550         self.sdk_branch = None
551         self.platform = "gtk"
552         self.build_type = "Release"
553         self.manifest_path = None
554         self.name = None
555         self.build_name = None
556         self.flatpak_root_path = None
557         self.cache_path = None
558         self.app_module = None
559         self.flatpak_default_args = []
560         self.check_available = False
561         self.wpe_extension = None
562
563         # Default application to run in the sandbox
564         self.command = None
565         self.user_command = []
566
567         # debug options
568         self.gdb = None
569         self.coredumpctl_matches = ""
570
571         # Extra build options
572         self.cmakeargs = ""
573         self.makeargs = ""
574
575         self.use_icecream = False
576         self.icc_version = None
577
578     def clean_args(self):
579         os.environ["FLATPAK_USER_DIR"] = os.environ.get("WEBKIT_FLATPAK_USER_DIR", os.path.realpath(os.path.join(scriptdir, "../../WebKitBuild", "UserFlatpak")))
580         try:
581             os.makedirs(os.environ["FLATPAK_USER_DIR"])
582         except OSError as e:
583             pass
584
585         configure_logging(logging.DEBUG if self.verbose else logging.INFO)
586         _log.debug("Using flatpak user dir: %s" % os.environ["FLATPAK_USER_DIR"])
587
588         if not self.debug and not self.release:
589             factory = PortFactory(SystemHost())
590             port = factory.get(self.platform)
591             self.debug = port.default_configuration() == "Debug"
592         self.build_type = "Debug" if self.debug else "Release"
593
594         self.platform = self.platform.upper()
595
596         if self.gdb is None and '--gdb' in sys.argv:
597             self.gdb = ""
598
599         self.command = "%s %s %s" % (os.path.join(self.sandbox_source_root,
600             "Tools/Scripts/run-minibrowser"),
601             "--" + self.platform.lower(),
602             " --debug" if self.debug else " --release")
603
604         self.name = "org.webkit.%s" % self.platform
605
606         if self.wpe_extension:
607             manifest_filename = WPE_MANIFEST_MAP[self.wpe_extension]
608         else:
609             manifest_filename = "org.webkit.WebKit.yaml"
610         self.manifest_path = os.path.abspath(os.path.join(scriptdir, '../flatpak/') + manifest_filename)
611
612         self.build_name = self.name + "-generated"
613
614         build_root = os.path.join(self.source_root, 'WebKitBuild')
615         self.flatpak_build_path = os.path.join(build_root, self.platform, "FlatpakTree" + self.build_type)
616         self.cache_path = os.path.join(build_root, "FlatpakCache")
617         self.build_path = os.path.join(build_root, self.platform, self.build_type)
618         try:
619             os.makedirs(self.build_path)
620         except OSError as e:
621             if e.errno != errno.EEXIST:
622                 raise e
623         self.config_file = os.path.join(self.build_path, 'webkit_flatpak_config.json')
624
625         Console.quiet = self.quiet
626         if not check_flatpak():
627             return False
628
629         repos = FlatpakRepos()
630         self.sdk_repo = repos.add(
631             FlatpakRepo("flathub",
632                         url="https://dl.flathub.org/repo/",
633                         repo_file="https://dl.flathub.org/repo/flathub.flatpakrepo"))
634
635         manifest = load_manifest(self.manifest_path, port_name=self.name)
636         if not manifest:
637             return False
638
639         self.app = manifest['app-id']
640
641         self.sdk_branch = manifest["runtime-version"]
642         self.finish_args = manifest.get("finish-args", [])
643         self.finish_args = remove_extension_points(self.finish_args)
644         self.runtime = FlatpakPackage(manifest['runtime'], self.sdk_branch,
645                                       self.sdk_repo, "x86_64",
646                                       hash=manifest.get("runtime-hash"))
647         self.locale = FlatpakPackage(manifest['runtime'] + '.Locale',
648                                      self.sdk_branch, self.sdk_repo, "x86_64")
649         self.sdk = FlatpakPackage(manifest['sdk'], self.sdk_branch,
650                                   self.sdk_repo, "x86_64",
651                                   hash=manifest.get("sdk-hash"))
652         self.packs = [self.runtime, self.locale, self.sdk]
653
654         if self.debug:
655             self.sdk_debug = FlatpakPackage(manifest['sdk'] + '.Debug', self.sdk_branch,
656                                       self.sdk_repo, "x86_64")
657             self.packs.append(self.sdk_debug)
658         self.manifest_generated_path = os.path.join(self.cache_path,
659                                                     self.build_name + ".json")
660         try:
661             with open(self.config_file) as config:
662                 json_config = json.load(config)
663                 self.icc_version = json_config['icecc_version']
664         except IOError as e:
665             pass
666
667         return True
668
669     def run_in_sandbox(self, *args, **kwargs):
670         cwd = kwargs.pop("cwd", None)
671         extra_env_vars = kwargs.pop("env", {})
672         stdout = kwargs.pop("stdout", sys.stdout)
673         extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
674
675         if not isinstance(args, list):
676             args = list(args)
677         if args:
678             if os.path.exists(args[0]):
679                 command = os.path.normpath(os.path.abspath(args[0]))
680                 # Take into account the fact that the webkit source dir is remounted inside the sandbox.
681                 args[0] = command.replace(self.source_root, self.sandbox_source_root)
682             if args[0].endswith("build-webkit"):
683                 args.append("--prefix=/app")
684
685         sandbox_build_path = os.path.join(self.sandbox_source_root, "WebKitBuild", self.build_type)
686         with tempfile.NamedTemporaryFile(mode="w") as tmpscript:
687             flatpak_command = ["flatpak", "build", "--die-with-parent",
688                 "--bind-mount=/run/shm=/dev/shm",
689                 # Workaround for https://webkit.org/b/187384 to have our own perl modules usable inside the sandbox
690                 # as setting the PERL5LIB envvar won't work inside apache (and for scripts using `perl -T``).
691                 "--bind-mount=/etc/perl=%s" % os.path.join(self.flatpak_build_path, "files/lib/perl"),
692                 "--bind-mount=/run/host/%s=%s" % (tempfile.gettempdir(), tempfile.gettempdir()),
693                 "--bind-mount=%s=%s" % (self.sandbox_source_root, self.source_root),
694                 "--talk-name=org.a11y.Bus",
695                 "--talk-name=org.gtk.vfs",
696                 "--talk-name=org.gtk.vfs.*",
697                 # We mount WebKitBuild/PORTNAME/BuildType to /app/webkit/WebKitBuild/BuildType
698                 # so we can build WPE and GTK in a same source tree.
699                 "--bind-mount=%s=%s" % (sandbox_build_path, self.build_path)]
700
701             forwarded = {
702                 "WEBKIT_TOP_LEVEL": "/app/",
703                 "TEST_RUNNER_INJECTED_BUNDLE_FILENAME": "/app/webkit/lib/libTestRunnerInjectedBundle.so",
704                 "ICECC_VERSION": self.icc_version,
705             }
706
707             env_var_prefixes_to_keep = [
708                 "GST",
709                 "GTK",
710                 "G",
711                 "JSC",
712                 "WEBKIT",
713                 "WEBKIT2",
714                 "WPE",
715                 "GIGACAGE",
716             ]
717
718             env_vars_to_keep = [
719                 "JavaScriptCoreUseJIT",
720                 "Malloc",
721                 "WAYLAND_DISPLAY",
722                 "DISPLAY",
723                 "LANG",
724                 "NUMBER_OF_PROCESSORS",
725                 "CCACHE_PREFIX",
726                 "QML2_IMPORT_PATH",
727             ]
728
729             if self.use_icecream:
730                 _log.debug('Enabling the icecream compiler')
731                 forwarded["CCACHE_PREFIX"] = "icecc"
732                 if not os.environ.get('NUMBER_OF_PROCESSORS'):
733                     n_cores = multiprocessing.cpu_count() * 3
734                     _log.debug('Follow icecream recommendation for the number of cores to use: %d' % n_cores)
735                     forwarded["NUMBER_OF_PROCESSORS"] = n_cores
736
737             env_vars = os.environ
738             env_vars.update(extra_env_vars)
739             for envvar, value in env_vars.items():
740                 if envvar.split("_")[0] in env_var_prefixes_to_keep or envvar in env_vars_to_keep:
741                     forwarded[envvar] = value
742
743             for envvar, value in forwarded.items():
744                 flatpak_command.append("--env=%s=%s" % (envvar, value))
745
746             flatpak_command += self.finish_args + extra_flatpak_args + [self.flatpak_build_path]
747
748             shell_string = ""
749             if args:
750                 if cwd:
751                     shell_string = 'cd "%s" && "%s"' % (cwd, '" "'.join(args))
752                 else:
753                     shell_string = '"%s"' % ('" "'.join(args))
754             else:
755                 shell_string = self.command
756                 if self.args:
757                     shell_string += ' "%s"' % '" "'.join(self.args)
758
759             tmpscript.write(shell_string)
760             tmpscript.flush()
761
762             _log.debug('Running in sandbox: "%s" %s\n' % ('" "'.join(flatpak_command), shell_string))
763             flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
764
765             try:
766                 subprocess.check_call(flatpak_command, stdout=stdout)
767             except subprocess.CalledProcessError as e:
768                 sys.stderr.write(str(e) + "\n")
769                 return e.returncode
770             except KeyboardInterrupt:
771                 return 0
772
773         return 0
774
775     def run(self):
776         if self.check_available:
777             return 0
778
779         if not self.clean_args():
780             return 1
781
782         if self.clean:
783             if os.path.exists(self.flatpak_build_path):
784                 shutil.rmtree(self.flatpak_build_path)
785             if os.path.exists(self.build_path):
786                 shutil.rmtree(self.build_path)
787
788         if self.update:
789             Console.message("Updating Flatpak environment for %s (%s)" % (
790                 self.platform, self.build_type))
791             if not self.no_flatpak_update:
792                 self.update_all()
793
794         return self.setup_dev_env()
795
796     def has_environment(self):
797         return os.path.exists(os.path.join(self.build_path, self.flatpak_build_path))
798
799     def save_config(self):
800         with open(self.config_file, 'w') as config:
801             json_config = {'icecc_version': self.icc_version}
802             json.dump(json_config, config)
803
804     def setup_ccache(self):
805         for compiler in ["c++", "cc", "clang", "clang++", "g++", "gcc"]:
806             self.run_in_sandbox("ln", "-s", "../../usr/bin/ccache", compiler, cwd="/app/bin")
807
808     def setup_icecc(self):
809         with tempfile.NamedTemporaryFile() as tmpfile:
810             self.run_in_sandbox('icecc', '--build-native', stdout=tmpfile, cwd=self.build_path)
811             tmpfile.flush()
812             tmpfile.seek(0)
813             icc_version_filename, = re.findall(r'.*creating (.*)', tmpfile.read())
814             self.icc_version = os.path.join(self.build_path, icc_version_filename)
815
816     def setup_dev_env(self):
817         if not os.path.exists(os.path.join(self.build_path, self.flatpak_build_path)) \
818                 or self.update or self.build_all:
819             self.install_all()
820             Console.message("Building %s and dependencies in %s",
821                             self.name, self.flatpak_build_path)
822
823             # Create environment dirs if necessary
824             try:
825                 os.makedirs(os.path.dirname(self.manifest_generated_path))
826             except OSError as e:
827                 if e.errno != errno.EEXIST:
828                     raise e
829             if not expand_manifest(self.manifest_path, self.manifest_generated_path,
830                                    self.name, self.sandbox_source_root, self.command):
831                 return 1
832
833             builder_args = ["flatpak-builder", "--disable-rofiles-fuse", "--state-dir",
834                             self.cache_path, "--ccache", self.flatpak_build_path, "--force-clean",
835                             self.manifest_generated_path]
836             builder_args.append("--build-only")
837             builder_args.append("--stop-at=%s" % self.app)
838
839             subprocess.check_call(builder_args)
840             self.setup_ccache()
841             self.setup_icecc()
842             self.save_config()
843
844         build_type = "--debug" if self.debug else "--release"
845         if self.build_webkit:
846             builder = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/build-webkit'),
847                 build_type, '--' + self.platform.lower()]
848             if self.makeargs:
849                 builder.append("--makeargs=%s" % self.makeargs)
850             if self.cmakeargs:
851                 builder.append("--cmakeargs=%s" % self.cmakeargs)
852             Console.message("Building webkit")
853             res = self.run_in_sandbox(*builder)
854
855             if res:
856                 return res
857         else:
858             Console.message("Using %s prefix in %s", self.name, self.flatpak_build_path)
859
860         if self.run_tests is not None:
861             test_launcher = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/run-webkit-tests'),
862                 build_type, '--' + self.platform.lower()] + self.run_tests
863             return self.run_in_sandbox(*test_launcher)
864         elif self.gdb is not None:
865             return self.run_gdb()
866         elif self.user_command:
867             return self.run_in_sandbox(*self.user_command)
868         elif not self.update and not self.build_webkit:
869             return self.run_in_sandbox()
870
871         return 0
872
873     def install_all(self):
874         for package in self.packs:
875             if not package.is_installed(self.sdk_branch):
876                 package.install()
877
878     def run_gdb(self):
879         with disable_signals():
880             try:
881                 subprocess.check_output(['which', 'coredumpctl'])
882             except subprocess.CalledProcessError as e:
883                 sys.stderr.write("'coredumpctl' not present on the system, can't run. (%s)\n" % e)
884                 return e.returncode
885
886         # We need access to the host from the sandbox to run.
887         with tempfile.NamedTemporaryFile() as coredump:
888             with tempfile.NamedTemporaryFile() as stderr:
889                 subprocess.check_call(["coredumpctl", "dump"] + shlex.split(self.coredumpctl_matches),
890                                       stdout=coredump, stderr=stderr)
891
892                 with open(stderr.name, 'r') as stderrf:
893                     stderr = stderrf.read()
894                 executable, = re.findall(".*Executable: (.*)", stderr)
895                 if not executable.startswith("/newroot"):
896                     sys.stderr.write("Executable %s doesn't seem to be a flatpaked application.\n" % executable)
897
898                 executable = executable.replace("/newroot", "")
899                 args = ["gdb", executable, "/run/host/%s" % coredump.name] + shlex.split(self.gdb)
900
901                 return self.run_in_sandbox(*args)
902
903     def update_all(self):
904         for m in [self.runtime, self.sdk, self.sdk_debug]:
905             if m:
906                 m.update()
907
908
909 def is_sandboxed():
910     return os.path.exists("/usr/manifest.json")
911
912
913 def run_in_sandbox_if_available(args):
914     if is_sandboxed():
915         return None
916
917     if not check_flatpak(verbose=False):
918         return None
919
920     flatpak_runner = WebkitFlatpak.load_from_args(args, add_help=False)
921     if not flatpak_runner.clean_args():
922         return None
923
924     if not flatpak_runner.has_environment():
925         return None
926
927     sys.exit(flatpak_runner.run_in_sandbox(*args))