35d48cdf62f0f96a92a1289469f9dc22980704af
[WebKit-https.git] / Tools / Scripts / webkitpy / port / mac.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (C) 2012-2019 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31 import os
32 import re
33
34 from webkitpy.common.memoized import memoized
35 from webkitpy.common.system.executive import ScriptError
36 from webkitpy.common.version import Version
37 from webkitpy.common.version_name_map import PUBLIC_TABLE, INTERNAL_TABLE
38 from webkitpy.common.version_name_map import VersionNameMap
39 from webkitpy.port.config import apple_additions, Config
40 from webkitpy.port.darwin import DarwinPort
41
42 _log = logging.getLogger(__name__)
43
44
45 class MacPort(DarwinPort):
46     port_name = "mac"
47
48     CURRENT_VERSION = Version(10, 15)
49
50     SDK = 'macosx'
51
52     ARCHITECTURES = ['x86_64', 'x86']
53
54     DEFAULT_ARCHITECTURE = 'x86_64'
55
56     def __init__(self, host, port_name, **kwargs):
57         DarwinPort.__init__(self, host, port_name, **kwargs)
58         version_name_map = VersionNameMap.map(host.platform)
59         self._os_version = None
60         split_port_name = port_name.split('-')
61         if len(split_port_name) > 1 and split_port_name[1] != 'wk2':
62             self._os_version = version_name_map.from_name(split_port_name[1])[1]
63         elif self.host.platform.is_mac() and apple_additions():
64             self._os_version = self.host.platform.os_version
65         if not self._os_version:
66             self._os_version = MacPort.CURRENT_VERSION
67         assert self._os_version.major == 10
68
69     def _build_driver_flags(self):
70         return ['ARCHS=i386'] if self.architecture() == 'x86' else []
71
72     def default_baseline_search_path(self, **kwargs):
73         versions_to_fallback = []
74         version_name_map = VersionNameMap.map(self.host.platform)
75
76         if self._os_version == self.CURRENT_VERSION:
77             versions_to_fallback = [self.CURRENT_VERSION]
78         else:
79             temp_version = Version(self._os_version.major, self._os_version.minor)
80             while temp_version != self.CURRENT_VERSION:
81                 versions_to_fallback.append(Version.from_iterable(temp_version))
82                 if temp_version < self.CURRENT_VERSION:
83                     temp_version.minor += 1
84                 else:
85                     temp_version.minor -= 1
86         wk_string = 'wk1'
87         if self.get_option('webkit_test_runner'):
88             wk_string = 'wk2'
89
90         expectations = []
91         for version in versions_to_fallback:
92             version_name = version_name_map.to_name(version, platform=self.port_name)
93             if version_name:
94                 standardized_version_name = version_name.lower().replace(' ', '')
95             apple_name = None
96             if apple_additions():
97                 apple_name = version_name_map.to_name(version, platform=self.port_name, table=INTERNAL_TABLE)
98
99             if apple_name:
100                 expectations.append(self._apple_baseline_path('mac-{}-{}'.format(apple_name.lower().replace(' ', ''), wk_string)))
101             if version_name:
102                 expectations.append(self._webkit_baseline_path('mac-{}-{}'.format(standardized_version_name, wk_string)))
103             if apple_name:
104                 expectations.append(self._apple_baseline_path('mac-{}'.format(apple_name.lower().replace(' ', ''))))
105             if version_name:
106                 expectations.append(self._webkit_baseline_path('mac-{}'.format(standardized_version_name)))
107
108         if apple_additions():
109             expectations.append(self._apple_baseline_path('{}-{}'.format(self.port_name, wk_string)))
110         expectations.append(self._webkit_baseline_path('{}-{}'.format(self.port_name, wk_string)))
111         if apple_additions():
112             expectations.append(self._apple_baseline_path('{}'.format(self.port_name)))
113         expectations.append(self._webkit_baseline_path(self.port_name))
114
115         if self.get_option('webkit_test_runner'):
116             expectations.append(self._webkit_baseline_path('wk2'))
117         return expectations
118
119     @memoized
120     def configuration_specifier_macros(self):
121         config_map = {}
122         version_name_map = VersionNameMap.map(self.host.platform)
123         for version in self._allowed_versions():
124             version_names = []
125             for newer in self._allowed_versions()[self._allowed_versions().index(version):]:
126                 version_name = version_name_map.to_name(newer, platform=self.port_name)
127                 if not version_name:
128                     version_name = version_name_map.to_name(newer, platform=self.port_name, table=INTERNAL_TABLE)
129                 version_names.append(version_name.lower().replace(' ', ''))
130             for table in [PUBLIC_TABLE, INTERNAL_TABLE]:
131                 version_name = version_name_map.to_name(version, platform=self.port_name, table=table)
132                 if not version_name:
133                     continue
134                 config_map[version_name.lower().replace(' ', '') + '+'] = version_names
135         return config_map
136
137     def environment_for_api_tests(self):
138         result = super(MacPort, self).environment_for_api_tests()
139         if self.get_option('guard_malloc'):
140             result['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib'
141             result['__XPC_DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib'
142         return result
143
144     def setup_environ_for_server(self, server_name=None):
145         env = super(MacPort, self).setup_environ_for_server(server_name)
146         if server_name == self.driver_name():
147             if self.get_option('leaks'):
148                 env['MallocStackLogging'] = '1'
149                 env['__XPC_MallocStackLogging'] = '1'
150                 env['MallocScribble'] = '1'
151                 env['__XPC_MallocScribble'] = '1'
152             if self.get_option('guard_malloc'):
153                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
154                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
155             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
156         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
157         return env
158
159     def _clear_global_caches_and_temporary_files(self):
160         self._filesystem.rmtree(os.path.expanduser('~/Library/' + self.driver_name()))
161         self._filesystem.rmtree(os.path.expanduser('~/Library/Application Support/' + self.driver_name()))
162         self._filesystem.rmtree(os.path.expanduser('~/Library/Caches/' + self.driver_name()))
163         self._filesystem.rmtree(os.path.expanduser('~/Library/WebKit/' + self.driver_name()))
164
165     def _path_to_user_cache_directory(self, suffix=None):
166         DIRHELPER_USER_DIR_SUFFIX = 'DIRHELPER_USER_DIR_SUFFIX'
167         CS_DARWIN_USER_CACHE_DIR = 65538
168
169         # The environment variable DIRHELPER_USER_DIR_SUFFIX is only honored on systems with
170         # System Integrity Protection disabled or with an Apple-Internal OS. To make this code
171         # work for all system configurations we compute the path with respect to the suffix
172         # by hand and temporarily unset the environment variable DIRHELPER_USER_DIR_SUFFIX (if set)
173         # to avoid it influencing confstr() on systems that honor DIRHELPER_USER_DIR_SUFFIX.
174         saved_suffix = None
175         if DIRHELPER_USER_DIR_SUFFIX in os.environ:
176             saved_suffix = os.environ[DIRHELPER_USER_DIR_SUFFIX]
177             del os.environ[DIRHELPER_USER_DIR_SUFFIX]
178         result = os.path.join(os.confstr(CS_DARWIN_USER_CACHE_DIR), suffix or '')
179         if saved_suffix is not None:
180             os.environ[DIRHELPER_USER_DIR_SUFFIX] = saved_suffix
181         return result
182
183     def operating_system(self):
184         return 'mac'
185
186     def default_child_processes(self, **kwargs):
187         default_count = super(MacPort, self).default_child_processes()
188
189         # FIXME: https://bugs.webkit.org/show_bug.cgi?id=95906  With too many WebProcess WK2 tests get stuck in resource contention.
190         # To alleviate the issue reduce the number of running processes
191         # Anecdotal evidence suggests that a 4 core/8 core logical machine may run into this, but that a 2 core/4 core logical machine does not.
192         should_throttle_for_wk2 = self.get_option('webkit_test_runner') and default_count > 4
193         # We also want to throttle for leaks bots.
194         if should_throttle_for_wk2 or self.get_option('leaks'):
195             default_count = int(.75 * default_count)
196
197         if should_throttle_for_wk2 and self.get_option('guard_malloc'):
198             # Some 12 core Macs get a lot of tests time out when running 18 WebKitTestRunner processes (it's not clear what this depends on).
199             # <rdar://problem/25750302>
200             default_count = min(default_count, 12)
201
202         # Make sure we have enough ram to support that many instances:
203         total_memory = self.host.platform.total_bytes_memory()
204         if total_memory:
205             bytes_per_drt = 256 * 1024 * 1024  # Assume each DRT needs 256MB to run.
206             overhead = 2048 * 1024 * 1024  # Assume we need 2GB free for the O/S
207             supportable_instances = max((total_memory - overhead) / bytes_per_drt, 1)  # Always use one process, even if we don't have space for it.
208             if supportable_instances < default_count:
209                 _log.warning("This machine could support %s child processes, but only has enough memory for %s." % (default_count, supportable_instances))
210         else:
211             _log.warning("Cannot determine available memory for child processes, using default child process count of %s." % default_count)
212             supportable_instances = default_count
213         return min(supportable_instances, default_count)
214
215     def start_helper(self, pixel_tests=False, prefer_integrated_gpu=False):
216         helper_path = self._path_to_helper()
217         if not helper_path:
218             _log.error("No path to LayoutTestHelper binary")
219             return False
220         _log.debug("Starting layout helper %s" % helper_path)
221         arguments = [helper_path, '--install-color-profile']
222         if prefer_integrated_gpu:
223             arguments.append('--prefer-integrated-gpu')
224         self._helper = self._executive.popen(arguments,
225             stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
226         is_ready = self._helper.stdout.readline()
227         if not is_ready.startswith(b'ready'):
228             _log.error("LayoutTestHelper could not start")
229             return False
230         return True
231
232     def reset_preferences(self):
233         _log.debug("Resetting persistent preferences")
234
235         for domain in ["DumpRenderTree", "WebKitTestRunner"]:
236             try:
237                 self._executive.run_command(["defaults", "delete", domain])
238             except ScriptError as e:
239                 # 'defaults' returns 1 if the domain did not exist
240                 if e.exit_code != 1:
241                     raise e
242
243     def stop_helper(self):
244         if self._helper:
245             _log.debug("Stopping LayoutTestHelper")
246             try:
247                 self._helper.stdin.write(b"x\n")
248                 self._helper.stdin.close()
249                 self._helper.wait()
250             except IOError as e:
251                 _log.debug("IOError raised while stopping helper: %s" % str(e))
252             self._helper = None
253
254     def logging_patterns_to_strip(self):
255         logging_patterns = []
256
257         # FIXME: Remove this after <rdar://problem/35954459> is fixed.
258         logging_patterns.append(('AVDCreateGPUAccelerator: Error loading GPU renderer\n', ''))
259
260         # FIXME: Remove this after <rdar://problem/51191120> is fixed.
261         logging_patterns.append((re.compile('GVA warning: getFreeDRMInstanceCount, maxDRMInstanceCount: .*\n'), ''))
262
263         # FIXME: Remove this after <rdar://problem/52897406> is fixed.
264         logging_patterns.append((re.compile('VPA info:.*\n'), ''))
265
266         return logging_patterns
267
268     def stderr_patterns_to_strip(self):
269         worthless_patterns = []
270         worthless_patterns.append((re.compile('.*(Fig|fig|itemasync|vt|mv_|PullParamSetSPS|ccrp_|client).* signalled err=.*\n'), ''))
271         worthless_patterns.append((re.compile('.*<<<< FigFilePlayer >>>>.*\n'), ''))
272         worthless_patterns.append((re.compile('.*<<<< FigFile >>>>.*\n'), ''))
273         worthless_patterns.append((re.compile('.*<<<< FAQ >>>>.*\n'), ''))
274         worthless_patterns.append((re.compile('.*<<<< MediaValidator >>>>.*\n'), ''))
275         worthless_patterns.append((re.compile('.*<<<< VMC >>>>.*\n'), ''))
276         worthless_patterns.append((re.compile('.*<<< FFR_Common >>>.*\n'), ''))
277         return worthless_patterns
278
279     def configuration_for_upload(self, host=None):
280         host = host or self.host
281         configuration = super(MacPort, self).configuration_for_upload(host=host)
282
283         output = host.executive.run_command(['/usr/sbin/sysctl', 'hw.model']).rstrip()
284         match = re.match(r'hw.model: (?P<model>.*)', output)
285         if match:
286             configuration['model'] = match.group('model')
287
288         return configuration
289
290
291 class MacCatalystPort(MacPort):
292     port_name = "maccatalyst"
293
294     def __init__(self, *args, **kwargs):
295         super(MacCatalystPort, self).__init__(*args, **kwargs)
296         self._config = Config(self._executive, self._filesystem, MacCatalystPort.port_name)
297
298     def _build_driver_flags(self):
299         return ['SDK_VARIANT=iosmac'] + super(MacCatalystPort, self)._build_driver_flags()