Hide next and previous form control buttons when WKWebView is editable
[WebKit-https.git] / Tools / flatpak / flatpakutils.py
1 # Copyright (C) 2017 Igalia S.L.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU Lesser General Public
5 # License as published by the Free Software Foundation; either
6 # version 2.1 of the License, or (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11 # Lesser General Public License for more details.
12 #
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this program; if not, write to the
15 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
16 # Boston, MA 02110-1301, USA.
17 import argparse
18 import logging
19 try:
20     import configparser
21 except ImportError:
22     import ConfigParser as configparser
23 from contextlib import contextmanager
24 import errno
25 import json
26 import multiprocessing
27 import os
28 import shlex
29 import shutil
30 import signal
31 import subprocess
32 import sys
33 import tempfile
34 import re
35
36 from webkitpy.common.system.systemhost import SystemHost
37 from webkitpy.port.factory import PortFactory
38 from webkitpy.common.system.logutils import configure_logging
39
40 try:
41     from urllib.parse import urlparse  # pylint: disable=E0611
42 except ImportError:
43     from urlparse import urlparse
44
45 try:
46     from urllib.request import urlopen  # pylint: disable=E0611
47 except ImportError:
48     from urllib2 import urlopen
49
50 FLATPAK_REQ = [
51     ("flatpak", "0.10.0"),
52     ("flatpak-builder", "0.10.0"),
53 ]
54
55 FLATPAK_VERSION = {}
56
57 WPE_MANIFEST_MAP = {
58     "qt": "org.webkit.WPEQT.yaml",
59 }
60
61 scriptdir = os.path.abspath(os.path.dirname(__file__))
62 _log = logging.getLogger(__name__)
63
64
65 class Colors:
66     HEADER = "\033[95m"
67     OKBLUE = "\033[94m"
68     OKGREEN = "\033[92m"
69     WARNING = "\033[93m"
70     FAIL = "\033[91m"
71     ENDC = "\033[0m"
72
73
74 class Console:
75
76     quiet = False
77
78     @classmethod
79     def message(cls, str_format, *args):
80         if cls.quiet:
81             return
82
83         if args:
84             print(str_format % args)
85         else:
86             print(str_format)
87
88         # Flush so that messages are printed at the right time
89         # as we use many subprocesses.
90         sys.stdout.flush()
91
92
93 def check_flatpak(verbose=True):
94     # Flatpak is only supported on Linux.
95     if not sys.platform.startswith("linux"):
96         return False
97
98     for app, required_version in FLATPAK_REQ:
99         try:
100             output = subprocess.check_output([app, "--version"])
101         except (subprocess.CalledProcessError, OSError):
102             if verbose:
103                 Console.message("\n%sYou need to install %s >= %s"
104                                 " to be able to use the '%s' script.\n\n"
105                                 "You can find some informations about"
106                                 " how to install it for your distribution at:\n"
107                                 "    * http://flatpak.org/%s\n", Colors.FAIL,
108                                 app, required_version, sys.argv[0], Colors.ENDC)
109             return False
110
111         def comparable_version(version):
112             return tuple(map(int, (version.split("."))))
113
114         version = output.decode("utf-8").split(" ")[1].strip("\n")
115         current = comparable_version(version)
116         FLATPAK_VERSION[app] = current
117         if current < comparable_version(required_version):
118             Console.message("\n%s%s %s required but %s found."
119                             " Please update and try again%s\n", Colors.FAIL,
120                             app, required_version, version, Colors.ENDC)
121             return False
122
123     return True
124
125
126 def remove_extension_points(array):
127     result_args = []
128     for arg in array:
129         if(not arg.startswith('--extension')):
130             result_args.append(arg)
131     return result_args
132
133
134 def remove_comments(string):
135     pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
136     # first group captures quoted strings (double or single)
137     # second group captures comments (//single-line or /* multi-line */)
138     regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
139
140     def _replacer(match):
141         # if the 2nd group (capturing comments) is not None,
142         # it means we have captured a non-quoted (real) comment string.
143         if match.group(2) is not None:
144             return ""  # so we will return empty to remove the comment
145         else:  # otherwise, we will return the 1st group
146             return match.group(1)  # captured quoted-string
147     return regex.sub(_replacer, string)
148
149
150 def load_manifest(manifest_path, port_name=None, command=None):
151     is_yaml = manifest_path.endswith('.yaml')
152     with open(manifest_path, "r") as mr:
153         contents = mr.read()
154
155         contents = contents % {"COMMAND": command, "PORTNAME": port_name}
156         if is_yaml:
157             import yaml
158
159             manifest = yaml.load(contents)
160         else:
161             contents = remove_comments(contents)
162             manifest = json.loads(contents)
163
164     return manifest
165
166
167 def expand_submodules_recurse(modules, manifest_path, port_name, command):
168     all_modules = []
169     if type(modules) is str:
170         submanifest_path = os.path.join(os.path.dirname(manifest_path), modules)
171         submanifest = load_manifest(submanifest_path, port_name=port_name, command=command)
172         all_modules.extend(expand_submodules_recurse(submanifest, submanifest_path, port_name, command))
173         return all_modules
174
175     # The last recurse manifest expand iteration might lead to a single module.
176     if not isinstance(modules, list):
177         all_modules.append(modules)
178         return all_modules
179
180     for module in modules:
181         if type(module) is str:
182             submanifest_path = os.path.join(os.path.dirname(manifest_path), module)
183             submanifest = load_manifest(submanifest_path, port_name=port_name, command=command)
184             all_modules.extend(expand_submodules_recurse(submanifest, submanifest_path, port_name, command))
185         else:
186             all_modules.append(module)
187
188     return all_modules
189
190 def expand_manifest(manifest_path, outfile, port_name, source_root, command):
191     """Creates the manifest file."""
192     try:
193         os.remove(outfile)
194     except OSError:
195         pass
196
197     manifest = load_manifest(manifest_path, port_name=port_name, command=command)
198     if not manifest:
199         return False
200
201     if "sdk-hash" in manifest:
202         del manifest["sdk-hash"]
203     if "runtime-hash" in manifest:
204         del manifest["runtime-hash"]
205
206     all_modules = []
207
208     overriden_modules = []
209     if "WEBKIT_EXTRA_MODULESETS" in os.environ:
210         overriden_modules = load_manifest(os.environ["WEBKIT_EXTRA_MODULESETS"])
211         if not overriden_modules:
212             overriden_modules = []
213     for modules in manifest["modules"]:
214         modules = expand_submodules_recurse(modules, manifest_path, port_name, command)
215
216         if not isinstance(modules, list):
217             modules = [modules]
218
219         for module in modules:
220             for overriden_module in overriden_modules:
221                 if module['name'] == overriden_module['name']:
222                     module = overriden_module
223                     overriden_modules.remove(module)
224                     break
225
226             all_modules.append(module)
227
228     # And add overriden modules right before the webkit port build def.
229     for overriden_module in overriden_modules:
230         all_modules.insert(-1, overriden_module)
231
232     manifest["modules"] = all_modules
233     for module in manifest["modules"]:
234         if not module.get("sources"):
235             continue
236
237         if module["sources"][0]["type"] == "git":
238             if port_name == module["name"]:
239                 repo = "file://" + source_root
240                 module["sources"][0]["url"] = repo
241
242         for source in module["sources"]:
243             if source["type"] == "patch" or (source["type"] == "file" and source.get('path')):
244                 source["path"] = os.path.join(os.path.dirname(manifest_path), source["path"])
245
246     with open(outfile, "w") as of:
247         of.write(json.dumps(manifest, indent=4))
248
249     return True
250
251
252 class FlatpakObject:
253
254     def __init__(self, user):
255         self.user = user
256
257     def flatpak(self, command, *args, **kwargs):
258         show_output = kwargs.pop("show_output", False)
259         comment = kwargs.pop("commend", None)
260         if comment:
261             Console.message(comment)
262
263         command = ["flatpak", command]
264         if self.user:
265             res = subprocess.check_output(command + ["--help"]).decode("utf-8")
266             if "--user" in res:
267                 command.append("--user")
268         command.extend(args)
269
270         if not show_output:
271             return subprocess.check_output(command).decode("utf-8")
272
273         return subprocess.check_call(command)
274
275
276 class FlatpakPackages(FlatpakObject):
277
278     def __init__(self, repos, user=True):
279         FlatpakObject.__init__(self, user=user)
280
281         self.repos = repos
282
283         self.runtimes = self.__detect_runtimes()
284         self.apps = self.__detect_apps()
285         self.packages = self.runtimes + self.apps
286
287
288     def __detect_packages(self, *args):
289         packs = []
290         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
291             out = self.flatpak("list", "-d", *args)
292             package_defs = [line for line in out.split("\n") if line]
293             for package_def in package_defs:
294                 splited_packaged_def = package_def.split()
295                 name, arch, branch = splited_packaged_def[0].split("/")
296
297                 # If installed from a file, the package is in no repo
298                 repo_name = splited_packaged_def[1]
299                 repo = self.repos.repos.get(repo_name)
300
301                 packs.append(FlatpakPackage(name, branch, repo, arch))
302         else:
303             out = self.flatpak("list", "--columns=application,arch,branch,origin", *args)
304             package_defs = [line for line in out.split("\n") if line]
305             for package_def in package_defs:
306                 name, arch, branch, origin = package_def.split("\t")
307
308                 # If installed from a file, the package is in no repo
309                 repo = self.repos.repos.get(origin)
310
311                 packs.append(FlatpakPackage(name, branch, repo, arch))
312
313         return packs
314
315
316     def __detect_runtimes(self):
317         return self.__detect_packages("--runtime")
318
319     def __detect_apps(self):
320         return self.__detect_packages()
321
322     def __iter__(self):
323         for package in self.packages:
324             yield package
325
326
327 class FlatpakRepos(FlatpakObject):
328
329     def __init__(self, user=True):
330         FlatpakObject.__init__(self, user=user)
331         self.repos = {}
332         self.update()
333
334     def update(self):
335         self.repos = {}
336         if FLATPAK_VERSION["flatpak"] < (1, 1, 2):
337             out = self.flatpak("remote-list", "-d")
338             remotes = [line for line in out.split("\n") if line]
339             for repo in remotes:
340                 for components in [repo.split(" "), repo.split("\t")]:
341                     if len(components) == 1:
342                         components = repo.split("\t")
343                     name = components[0]
344                     desc = ""
345                     url = None
346                     for elem in components[1:]:
347                         if not elem:
348                             continue
349                         parsed_url = urlparse(elem)
350                         if parsed_url.scheme:
351                             url = elem
352                             break
353
354                         if desc:
355                             desc += " "
356                         desc += elem
357
358                     if url:
359                         break
360
361                 if not url:
362                     Console.message("No valid URI found for: %s", repo)
363                     continue
364
365                 self.repos[name] = FlatpakRepo(name, url, desc, repos=self)
366         else:
367             out = self.flatpak("remote-list", "--columns=name,title,url")
368             remotes = [line for line in out.split("\n") if line]
369             for remote in remotes:
370                 name, title, url = remote.split("\t")
371                 parsed_url = urlparse(url)
372                 if not parsed_url.scheme:
373                     Console.message("No valid URI found for: %s", remote)
374                     continue
375
376                 self.repos[name] = FlatpakRepo(name, url, title, repos=self)
377
378         self.packages = FlatpakPackages(self)
379
380     def add(self, repo, override=True):
381         same_name = None
382         for name, tmprepo in self.repos.items():
383             if repo.url == tmprepo.url:
384                 return tmprepo
385             elif repo.name == name:
386                 same_name = tmprepo
387
388         if same_name:
389             if override:
390                 self.flatpak("remote-modify", repo.name, "--url=" + repo.url,
391                              comment="Setting repo %s URL from %s to %s"
392                              % (repo.name, same_name.url, repo.url))
393                 same_name.url = repo.url
394
395                 return same_name
396             else:
397                 return None
398         else:
399             self.flatpak("remote-add", repo.name, "--from", repo.repo_file.name,
400                          "--if-not-exists",
401                          comment="Adding repo %s" % repo.name)
402
403         repo.repos = self
404         return repo
405
406
407 class FlatpakRepo(FlatpakObject):
408
409     def __init__(self, name, desc=None, url=None,
410                  repo_file=None, user=True, repos=None):
411         FlatpakObject.__init__(self, user=user)
412
413         self.name = name
414         self.url = url
415         self.desc = desc
416         self.repo_file_name = repo_file
417         self._repo_file = None
418         self.repos = repos
419         assert name
420         if repo_file and not url:
421             repo = configparser.ConfigParser()
422             repo.read(self.repo_file.name)
423             self.url = repo["Flatpak Repo"]["Url"]
424         else:
425             assert url
426
427     @property
428     def repo_file(self):
429         if self._repo_file:
430             return self._repo_file
431
432         assert self.repo_file_name
433         self._repo_file = tempfile.NamedTemporaryFile(mode="wb")
434         self._repo_file.write(urlopen(self.repo_file_name).read())
435         self._repo_file.flush()
436
437         return self._repo_file
438
439
440 class FlatpakPackage(FlatpakObject):
441     """A flatpak app."""
442
443     def __init__(self, name, branch, repo, arch, user=True, hash=None):
444         FlatpakObject.__init__(self, user=user)
445
446         self.name = name
447         self.branch = str(branch)
448         self.repo = repo
449         self.arch = arch
450         self.hash = hash
451
452     def __str__(self):
453         return "%s/%s/%s %s" % (self.name, self.arch, self.branch, self.repo.name)
454
455     def is_installed(self, branch):
456         if not self.repo:
457             # Bundle installed from file
458             return True
459
460         self.repo.repos.update()
461         for package in self.repo.repos.packages:
462             if package.name == self.name and \
463                     package.branch == branch and \
464                     package.arch == self.arch:
465                 return True
466
467         return False
468
469     def install(self):
470         if not self.repo:
471             return False
472
473         args = ["install", self.repo.name, self.name, "--reinstall", self.branch, "--assumeyes"]
474
475         self.flatpak(*args, show_output=True,
476                      comment="Installing from " + self.repo.name + " " +
477                              self.name + " " + self.arch + " " + self.branch)
478
479     def update(self):
480         if not self.is_installed(self.branch):
481             return self.install()
482
483         extra_args = []
484         comment = "Updating %s" % self.name
485         if self.hash:
486             extra_args = ["--commit", self.hash]
487             comment += " to %s" % self.hash
488
489         extra_args.append("--assumeyes")
490
491         self.flatpak("update", self.name, self.branch, show_output=True,
492                     *extra_args, comment=comment)
493
494
495 @contextmanager
496 def disable_signals(signals=[signal.SIGINT]):
497     old_signal_handlers = []
498
499     for disabled_signal in signals:
500         old_signal_handlers.append((disabled_signal, signal.getsignal(disabled_signal)))
501         signal.signal(disabled_signal, signal.SIG_IGN)
502
503     yield
504
505     for disabled_signal, previous_handler in old_signal_handlers:
506         signal.signal(disabled_signal, previous_handler)
507
508
509 class WebkitFlatpak:
510
511     @staticmethod
512     def load_from_args(args=None, add_help=True):
513         self = WebkitFlatpak()
514
515         parser = argparse.ArgumentParser(prog="webkit-flatpak", add_help=add_help)
516         general = parser.add_argument_group("General")
517         general.add_argument('--verbose', action='store_true',
518                              help='Show debug message')
519         general.add_argument("--debug",
520                             help="Compile with Debug configuration, also installs Sdk debug symboles.",
521                             action="store_true")
522         general.add_argument("--release", help="Compile with Release configuration.", action="store_true")
523         general.add_argument('--platform', action='store', help='Platform to use (e.g., "mac-lion")'),
524         general.add_argument('--gtk', action='store_const', dest='platform', const='gtk',
525                              help='Alias for --platform=gtk')
526         general.add_argument('--wpe', action='store_const', dest='platform', const='wpe',
527                             help=('Alias for --platform=wpe'))
528         general.add_argument("-nf", "--no-flatpak-update", dest="no_flatpak_update",
529                             action="store_true",
530                             help="Do not update flaptak runtime/sdk")
531         general.add_argument("-u", "--update", dest="update",
532                             action="store_true",
533                             help="Update the runtime/sdk/app and rebuild the development environment if needed")
534         general.add_argument("-b", "--build-webkit", dest="build_webkit",
535                             action="store_true",
536                             help="Force rebuilding the app.")
537         general.add_argument("-ba", "--build-all", dest="build_all",
538                             action="store_true",
539                             help="Force rebuilding the app and its dependencies.")
540         general.add_argument("-q", "--quiet", dest="quiet",
541                             action="store_true",
542                             help="Do not print anything")
543         general.add_argument("-t", "--tests", dest="run_tests",
544                             nargs=argparse.REMAINDER,
545                             help="Run LayoutTests")
546         general.add_argument("-c", "--command",
547                             nargs=argparse.REMAINDER,
548                             help="The command to run in the sandbox",
549                             dest="user_command")
550         general.add_argument('--available', action='store_true', dest="check_available", help='Check if required dependencies are available.'),
551         general.add_argument("--use-icecream", help="Use the distributed icecream (icecc) compiler.", action="store_true")
552         general.add_argument("--wpe-extension", action="store", dest="wpe_extension", help="WPE Extension to enable")
553
554         debugoptions = parser.add_argument_group("Debugging")
555         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
556         debugoptions.add_argument("-m", "--coredumpctl-matches", default="", help='Arguments to pass to gdb.')
557
558         buildoptions = parser.add_argument_group("Extra build arguments")
559         buildoptions.add_argument("--makeargs", help="Optional Makefile flags")
560         buildoptions.add_argument("--cmakeargs",
561                                 help="One or more optional CMake flags (e.g. --cmakeargs=\"-DFOO=bar -DCMAKE_PREFIX_PATH=/usr/local\")")
562
563         general.add_argument("--clean", dest="clean", action="store_true",
564             help="Clean previous builds and restart from scratch")
565
566         _, self.args = parser.parse_known_args(args=args, namespace=self)
567
568         return self
569
570     def __init__(self):
571         self.sdk_repo = None
572         self.runtime = None
573         self.locale = None
574         self.sdk = None
575         self.sdk_debug = None
576         self.app = None
577
578         self.verbose = False
579         self.quiet = False
580         self.packs = []
581         self.update = False
582         self.args = []
583         self.finish_args = None
584
585         self.no_flatpak_update = False
586         self.release = False
587         self.debug = False
588         self.clean = False
589         self.run_tests = None
590         self.source_root = os.path.normpath(os.path.abspath(os.path.join(scriptdir, '../../')))
591         # Where the source folder is mounted inside the sandbox.
592         self.sandbox_source_root = "/app/webkit"
593
594         self.build_webkit = False
595         self.build_all = False
596
597         self.sdk_branch = None
598         self.platform = "gtk"
599         self.build_type = "Release"
600         self.manifest_path = None
601         self.name = None
602         self.build_name = None
603         self.flatpak_root_path = None
604         self.cache_path = None
605         self.app_module = None
606         self.flatpak_default_args = []
607         self.check_available = False
608         self.wpe_extension = None
609
610         # Default application to run in the sandbox
611         self.command = None
612         self.user_command = []
613
614         # debug options
615         self.gdb = None
616         self.coredumpctl_matches = ""
617
618         # Extra build options
619         self.cmakeargs = ""
620         self.makeargs = ""
621
622         self.use_icecream = False
623         self.icc_version = None
624
625     def clean_args(self):
626         os.environ["FLATPAK_USER_DIR"] = os.environ.get("WEBKIT_FLATPAK_USER_DIR", os.path.realpath(os.path.join(scriptdir, "../../WebKitBuild", "UserFlatpak")))
627         try:
628             os.makedirs(os.environ["FLATPAK_USER_DIR"])
629         except OSError as e:
630             pass
631
632         configure_logging(logging.DEBUG if self.verbose else logging.INFO)
633         _log.debug("Using flatpak user dir: %s" % os.environ["FLATPAK_USER_DIR"])
634
635         if not self.debug and not self.release:
636             factory = PortFactory(SystemHost())
637             port = factory.get(self.platform)
638             self.debug = port.default_configuration() == "Debug"
639         self.build_type = "Debug" if self.debug else "Release"
640
641         self.platform = self.platform.upper()
642
643         if self.gdb is None and '--gdb' in sys.argv:
644             self.gdb = ""
645
646         self.command = "%s %s %s" % (os.path.join(self.sandbox_source_root,
647             "Tools/Scripts/run-minibrowser"),
648             "--" + self.platform.lower(),
649             " --debug" if self.debug else " --release")
650
651         self.name = "org.webkit.%s" % self.platform
652
653         if self.wpe_extension:
654             manifest_filename = WPE_MANIFEST_MAP[self.wpe_extension]
655         else:
656             manifest_filename = "org.webkit.WebKit.yaml"
657         self.manifest_path = os.path.abspath(os.path.join(scriptdir, '../flatpak/') + manifest_filename)
658
659         self.build_name = self.name + "-generated"
660
661         build_root = os.path.join(self.source_root, 'WebKitBuild')
662         self.flatpak_build_path = os.path.join(build_root, self.platform, "FlatpakTree" + self.build_type)
663         self.cache_path = os.path.join(build_root, "FlatpakCache")
664         self.build_path = os.path.join(build_root, self.platform, self.build_type)
665         try:
666             os.makedirs(self.build_path)
667         except OSError as e:
668             if e.errno != errno.EEXIST:
669                 raise e
670         self.config_file = os.path.join(self.build_path, 'webkit_flatpak_config.json')
671
672         Console.quiet = self.quiet
673         if not check_flatpak():
674             return False
675
676         repos = FlatpakRepos()
677         self.sdk_repo = repos.add(
678             FlatpakRepo("flathub",
679                         url="https://dl.flathub.org/repo/",
680                         repo_file="https://dl.flathub.org/repo/flathub.flatpakrepo"))
681
682         manifest = load_manifest(self.manifest_path, port_name=self.name)
683         if not manifest:
684             return False
685
686         self.app = manifest['app-id']
687
688         self.sdk_branch = manifest["runtime-version"]
689         self.finish_args = manifest.get("finish-args", [])
690         self.finish_args = remove_extension_points(self.finish_args)
691         self.runtime = FlatpakPackage(manifest['runtime'], self.sdk_branch,
692                                       self.sdk_repo, "x86_64",
693                                       hash=manifest.get("runtime-hash"))
694         self.locale = FlatpakPackage(manifest['runtime'] + '.Locale',
695                                      self.sdk_branch, self.sdk_repo, "x86_64")
696         self.sdk = FlatpakPackage(manifest['sdk'], self.sdk_branch,
697                                   self.sdk_repo, "x86_64",
698                                   hash=manifest.get("sdk-hash"))
699         self.packs = [self.runtime, self.locale, self.sdk]
700
701         if self.debug:
702             self.sdk_debug = FlatpakPackage(manifest['sdk'] + '.Debug', self.sdk_branch,
703                                       self.sdk_repo, "x86_64")
704             self.packs.append(self.sdk_debug)
705         self.manifest_generated_path = os.path.join(self.cache_path,
706                                                     self.build_name + ".json")
707         try:
708             with open(self.config_file) as config:
709                 json_config = json.load(config)
710                 self.icc_version = json_config['icecc_version']
711         except IOError as e:
712             pass
713
714         return True
715
716     def run_in_sandbox(self, *args, **kwargs):
717         cwd = kwargs.pop("cwd", None)
718         extra_env_vars = kwargs.pop("env", {})
719         stdout = kwargs.pop("stdout", sys.stdout)
720         extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
721
722         if not isinstance(args, list):
723             args = list(args)
724         if args:
725             if os.path.exists(args[0]):
726                 command = os.path.normpath(os.path.abspath(args[0]))
727                 # Take into account the fact that the webkit source dir is remounted inside the sandbox.
728                 args[0] = command.replace(self.source_root, self.sandbox_source_root)
729             if args[0].endswith("build-webkit"):
730                 args.append("--prefix=/app")
731
732         sandbox_build_path = os.path.join(self.sandbox_source_root, "WebKitBuild", self.build_type)
733         with tempfile.NamedTemporaryFile(mode="w") as tmpscript:
734             flatpak_command = ["flatpak", "build", "--die-with-parent",
735                 "--bind-mount=/run/shm=/dev/shm",
736                 # Workaround for https://webkit.org/b/187384 to have our own perl modules usable inside the sandbox
737                 # as setting the PERL5LIB envvar won't work inside apache (and for scripts using `perl -T``).
738                 "--bind-mount=/etc/perl=%s" % os.path.join(self.flatpak_build_path, "files/lib/perl"),
739                 "--bind-mount=/run/host/%s=%s" % (tempfile.gettempdir(), tempfile.gettempdir()),
740                 "--bind-mount=%s=%s" % (self.sandbox_source_root, self.source_root),
741                 "--talk-name=org.a11y.Bus",
742                 "--talk-name=org.gtk.vfs",
743                 "--talk-name=org.gtk.vfs.*",
744                 # We mount WebKitBuild/PORTNAME/BuildType to /app/webkit/WebKitBuild/BuildType
745                 # so we can build WPE and GTK in a same source tree.
746                 "--bind-mount=%s=%s" % (sandbox_build_path, self.build_path)]
747
748             forwarded = {
749                 "WEBKIT_TOP_LEVEL": "/app/",
750                 "TEST_RUNNER_INJECTED_BUNDLE_FILENAME": "/app/webkit/lib/libTestRunnerInjectedBundle.so",
751                 "ICECC_VERSION": self.icc_version,
752             }
753
754             env_var_prefixes_to_keep = [
755                 "GST",
756                 "GTK",
757                 "G",
758                 "JSC",
759                 "WEBKIT",
760                 "WEBKIT2",
761                 "WPE",
762                 "GIGACAGE",
763             ]
764
765             env_vars_to_keep = [
766                 "JavaScriptCoreUseJIT",
767                 "Malloc",
768                 "WAYLAND_DISPLAY",
769                 "DISPLAY",
770                 "LANG",
771                 "NUMBER_OF_PROCESSORS",
772                 "CCACHE_PREFIX",
773                 "QML2_IMPORT_PATH",
774             ]
775
776             if self.use_icecream:
777                 _log.debug('Enabling the icecream compiler')
778                 forwarded["CCACHE_PREFIX"] = "icecc"
779                 if not os.environ.get('NUMBER_OF_PROCESSORS'):
780                     n_cores = multiprocessing.cpu_count() * 3
781                     _log.debug('Follow icecream recommendation for the number of cores to use: %d' % n_cores)
782                     forwarded["NUMBER_OF_PROCESSORS"] = n_cores
783
784             env_vars = os.environ
785             env_vars.update(extra_env_vars)
786             for envvar, value in env_vars.items():
787                 if envvar.split("_")[0] in env_var_prefixes_to_keep or envvar in env_vars_to_keep:
788                     forwarded[envvar] = value
789
790             for envvar, value in forwarded.items():
791                 flatpak_command.append("--env=%s=%s" % (envvar, value))
792
793             flatpak_command += self.finish_args + extra_flatpak_args + [self.flatpak_build_path]
794
795             shell_string = ""
796             if args:
797                 if cwd:
798                     shell_string = 'cd "%s" && "%s"' % (cwd, '" "'.join(args))
799                 else:
800                     shell_string = '"%s"' % ('" "'.join(args))
801             else:
802                 shell_string = self.command
803                 if self.args:
804                     shell_string += ' "%s"' % '" "'.join(self.args)
805
806             tmpscript.write(shell_string)
807             tmpscript.flush()
808
809             _log.debug('Running in sandbox: "%s" %s\n' % ('" "'.join(flatpak_command), shell_string))
810             flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
811
812             try:
813                 subprocess.check_call(flatpak_command, stdout=stdout)
814             except subprocess.CalledProcessError as e:
815                 sys.stderr.write(str(e) + "\n")
816                 return e.returncode
817             except KeyboardInterrupt:
818                 return 0
819
820         return 0
821
822     def run(self):
823         if self.check_available:
824             return 0
825
826         if not self.clean_args():
827             return 1
828
829         if self.clean:
830             if os.path.exists(self.flatpak_build_path):
831                 shutil.rmtree(self.flatpak_build_path)
832             if os.path.exists(self.build_path):
833                 shutil.rmtree(self.build_path)
834
835         if self.update:
836             Console.message("Updating Flatpak environment for %s (%s)" % (
837                 self.platform, self.build_type))
838             if not self.no_flatpak_update:
839                 self.update_all()
840
841         return self.setup_dev_env()
842
843     def has_environment(self):
844         return os.path.exists(os.path.join(self.build_path, self.flatpak_build_path))
845
846     def save_config(self):
847         with open(self.config_file, 'w') as config:
848             json_config = {'icecc_version': self.icc_version}
849             json.dump(json_config, config)
850
851     def setup_ccache(self):
852         for compiler in ["c++", "cc", "clang", "clang++", "g++", "gcc"]:
853             self.run_in_sandbox("ln", "-s", "../../usr/bin/ccache", compiler, cwd="/app/bin")
854
855     def setup_icecc(self):
856         with tempfile.NamedTemporaryFile() as tmpfile:
857             self.run_in_sandbox('icecc', '--build-native', stdout=tmpfile, cwd=self.build_path)
858             tmpfile.flush()
859             tmpfile.seek(0)
860             icc_version_filename, = re.findall(r'.*creating (.*)', tmpfile.read())
861             self.icc_version = os.path.join(self.build_path, icc_version_filename)
862
863     def setup_dev_env(self):
864         if not os.path.exists(os.path.join(self.build_path, self.flatpak_build_path)) \
865                 or self.update or self.build_all:
866             self.install_all()
867             Console.message("Building %s and dependencies in %s",
868                             self.name, self.flatpak_build_path)
869
870             # Create environment dirs if necessary
871             try:
872                 os.makedirs(os.path.dirname(self.manifest_generated_path))
873             except OSError as e:
874                 if e.errno != errno.EEXIST:
875                     raise e
876             if not expand_manifest(self.manifest_path, self.manifest_generated_path,
877                                    self.name, self.sandbox_source_root, self.command):
878                 return 1
879
880             builder_args = ["flatpak-builder", "--disable-rofiles-fuse", "--state-dir",
881                             self.cache_path, "--ccache", self.flatpak_build_path, "--force-clean",
882                             self.manifest_generated_path]
883             builder_args.append("--build-only")
884             builder_args.append("--stop-at=%s" % self.app)
885
886             subprocess.check_call(builder_args)
887             self.setup_ccache()
888             self.setup_icecc()
889             self.save_config()
890
891         build_type = "--debug" if self.debug else "--release"
892         if self.build_webkit:
893             builder = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/build-webkit'),
894                 build_type, '--' + self.platform.lower()]
895             if self.makeargs:
896                 builder.append("--makeargs=%s" % self.makeargs)
897             if self.cmakeargs:
898                 builder.append("--cmakeargs=%s" % self.cmakeargs)
899             Console.message("Building webkit")
900             res = self.run_in_sandbox(*builder)
901
902             if res:
903                 return res
904         else:
905             Console.message("Using %s prefix in %s", self.name, self.flatpak_build_path)
906
907         if self.run_tests is not None:
908             test_launcher = [os.path.join(self.sandbox_source_root, 'Tools/Scripts/run-webkit-tests'),
909                 build_type, '--' + self.platform.lower()] + self.run_tests
910             return self.run_in_sandbox(*test_launcher)
911         elif self.gdb is not None:
912             return self.run_gdb()
913         elif self.user_command:
914             return self.run_in_sandbox(*self.user_command)
915         elif not self.update and not self.build_webkit:
916             return self.run_in_sandbox()
917
918         return 0
919
920     def install_all(self):
921         for package in self.packs:
922             if not package.is_installed(self.sdk_branch):
923                 package.install()
924
925     def run_gdb(self):
926         with disable_signals():
927             try:
928                 subprocess.check_output(['which', 'coredumpctl'])
929             except subprocess.CalledProcessError as e:
930                 sys.stderr.write("'coredumpctl' not present on the system, can't run. (%s)\n" % e)
931                 return e.returncode
932
933         # We need access to the host from the sandbox to run.
934         with tempfile.NamedTemporaryFile() as coredump:
935             with tempfile.NamedTemporaryFile() as stderr:
936                 subprocess.check_call(["coredumpctl", "dump"] + shlex.split(self.coredumpctl_matches),
937                                       stdout=coredump, stderr=stderr)
938
939                 with open(stderr.name, 'r') as stderrf:
940                     stderr = stderrf.read()
941                 executable, = re.findall(".*Executable: (.*)", stderr)
942                 if not executable.startswith("/newroot"):
943                     sys.stderr.write("Executable %s doesn't seem to be a flatpaked application.\n" % executable)
944
945                 executable = executable.replace("/newroot", "")
946                 args = ["gdb", executable, "/run/host/%s" % coredump.name] + shlex.split(self.gdb)
947
948                 return self.run_in_sandbox(*args)
949
950     def update_all(self):
951         for m in [self.runtime, self.sdk, self.sdk_debug]:
952             if m:
953                 m.update()
954
955
956 def is_sandboxed():
957     return os.path.exists("/usr/manifest.json")
958
959
960 def run_in_sandbox_if_available(args):
961     if is_sandboxed():
962         return None
963
964     if not check_flatpak(verbose=False):
965         return None
966
967     flatpak_runner = WebkitFlatpak.load_from_args(args, add_help=False)
968     if not flatpak_runner.clean_args():
969         return None
970
971     if not flatpak_runner.has_environment():
972         return None
973
974     sys.exit(flatpak_runner.run_in_sandbox(*args))