2145d021613743f622d7d8dfcc2aed613d2febe9
[WebKit-https.git] / Tools / Scripts / webkitpy / port / mac.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (C) 2012-2019 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31 import os
32 import re
33
34 from webkitpy.common.memoized import memoized
35 from webkitpy.common.system.executive import ScriptError
36 from webkitpy.common.version import Version
37 from webkitpy.common.version_name_map import PUBLIC_TABLE, INTERNAL_TABLE
38 from webkitpy.common.version_name_map import VersionNameMap
39 from webkitpy.port.config import apple_additions
40 from webkitpy.port.darwin import DarwinPort
41
42 _log = logging.getLogger(__name__)
43
44
45 class MacPort(DarwinPort):
46     port_name = "mac"
47
48     CURRENT_VERSION = Version(10, 14)
49
50     SDK = 'macosx'
51
52     ARCHITECTURES = ['x86_64', 'x86']
53
54     DEFAULT_ARCHITECTURE = 'x86_64'
55
56     def __init__(self, host, port_name, **kwargs):
57         DarwinPort.__init__(self, host, port_name, **kwargs)
58         version_name_map = VersionNameMap.map(host.platform)
59         self._os_version = None
60         split_port_name = port_name.split('-')
61         if len(split_port_name) > 1 and split_port_name[1] != 'wk2':
62             self._os_version = version_name_map.from_name(split_port_name[1])[1]
63         elif self.host.platform.is_mac() and apple_additions():
64             self._os_version = self.host.platform.os_version
65         if not self._os_version:
66             self._os_version = MacPort.CURRENT_VERSION
67         assert self._os_version.major == 10
68
69     def _build_driver_flags(self):
70         return ['ARCHS=i386'] if self.architecture() == 'x86' else []
71
72     def default_baseline_search_path(self, **kwargs):
73         versions_to_fallback = []
74         version_name_map = VersionNameMap.map(self.host.platform)
75
76         if self._os_version == self.CURRENT_VERSION:
77             versions_to_fallback = [self.CURRENT_VERSION]
78         else:
79             temp_version = Version(self._os_version.major, self._os_version.minor)
80             while temp_version != self.CURRENT_VERSION:
81                 versions_to_fallback.append(Version.from_iterable(temp_version))
82                 if temp_version < self.CURRENT_VERSION:
83                     temp_version.minor += 1
84                 else:
85                     temp_version.minor -= 1
86         wk_string = 'wk1'
87         if self.get_option('webkit_test_runner'):
88             wk_string = 'wk2'
89
90         expectations = []
91         for version in versions_to_fallback:
92             version_name = version_name_map.to_name(version, platform=self.port_name)
93             if version_name:
94                 standardized_version_name = version_name.lower().replace(' ', '')
95             apple_name = None
96             if apple_additions():
97                 apple_name = version_name_map.to_name(version, platform=self.port_name, table=INTERNAL_TABLE)
98
99             if apple_name:
100                 expectations.append(self._apple_baseline_path('mac-{}-{}'.format(apple_name.lower().replace(' ', ''), wk_string)))
101             if version_name:
102                 expectations.append(self._webkit_baseline_path('mac-{}-{}'.format(standardized_version_name, wk_string)))
103             if apple_name:
104                 expectations.append(self._apple_baseline_path('mac-{}'.format(apple_name.lower().replace(' ', ''))))
105             if version_name:
106                 expectations.append(self._webkit_baseline_path('mac-{}'.format(standardized_version_name)))
107
108         if apple_additions():
109             expectations.append(self._apple_baseline_path('{}-{}'.format(self.port_name, wk_string)))
110         expectations.append(self._webkit_baseline_path('{}-{}'.format(self.port_name, wk_string)))
111         if apple_additions():
112             expectations.append(self._apple_baseline_path('{}'.format(self.port_name)))
113         expectations.append(self._webkit_baseline_path(self.port_name))
114
115         if self.get_option('webkit_test_runner'):
116             expectations.append(self._webkit_baseline_path('wk2'))
117         return expectations
118
119     @memoized
120     def configuration_specifier_macros(self):
121         config_map = {}
122         version_name_map = VersionNameMap.map(self.host.platform)
123         for version in self._allowed_versions():
124             version_names = []
125             for newer in self._allowed_versions()[self._allowed_versions().index(version):]:
126                 version_name = version_name_map.to_name(newer, platform=self.port_name)
127                 if not version_name:
128                     version_name = version_name_map.to_name(newer, platform=self.port_name, table=INTERNAL_TABLE)
129                 version_names.append(version_name.lower().replace(' ', ''))
130             for table in [PUBLIC_TABLE, INTERNAL_TABLE]:
131                 version_name = version_name_map.to_name(version, platform=self.port_name, table=table)
132                 if not version_name:
133                     continue
134                 config_map[version_name.lower().replace(' ', '') + '+'] = version_names
135         return config_map
136
137     def environment_for_api_tests(self):
138         result = super(MacPort, self).environment_for_api_tests()
139         if self.get_option('guard_malloc'):
140             result['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib'
141             result['__XPC_DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib'
142         return result
143
144     def setup_environ_for_server(self, server_name=None):
145         env = super(MacPort, self).setup_environ_for_server(server_name)
146         if server_name == self.driver_name():
147             if self.get_option('leaks'):
148                 env['MallocStackLogging'] = '1'
149                 env['__XPC_MallocStackLogging'] = '1'
150                 env['MallocScribble'] = '1'
151                 env['__XPC_MallocScribble'] = '1'
152             if self.get_option('guard_malloc'):
153                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
154                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
155             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
156         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
157         return env
158
159     def _clear_global_caches_and_temporary_files(self):
160         self._filesystem.rmtree(os.path.expanduser('~/Library/' + self.driver_name()))
161         self._filesystem.rmtree(os.path.expanduser('~/Library/Application Support/' + self.driver_name()))
162         self._filesystem.rmtree(os.path.expanduser('~/Library/Caches/' + self.driver_name()))
163         self._filesystem.rmtree(os.path.expanduser('~/Library/WebKit/' + self.driver_name()))
164
165     def _path_to_user_cache_directory(self, suffix=None):
166         DIRHELPER_USER_DIR_SUFFIX = 'DIRHELPER_USER_DIR_SUFFIX'
167         CS_DARWIN_USER_CACHE_DIR = 65538
168
169         # The environment variable DIRHELPER_USER_DIR_SUFFIX is only honored on systems with
170         # System Integrity Protection disabled or with an Apple-Internal OS. To make this code
171         # work for all system configurations we compute the path with respect to the suffix
172         # by hand and temporarily unset the environment variable DIRHELPER_USER_DIR_SUFFIX (if set)
173         # to avoid it influencing confstr() on systems that honor DIRHELPER_USER_DIR_SUFFIX.
174         saved_suffix = None
175         if DIRHELPER_USER_DIR_SUFFIX in os.environ:
176             saved_suffix = os.environ[DIRHELPER_USER_DIR_SUFFIX]
177             del os.environ[DIRHELPER_USER_DIR_SUFFIX]
178         result = os.path.join(os.confstr(CS_DARWIN_USER_CACHE_DIR), suffix or '')
179         if saved_suffix is not None:
180             os.environ[DIRHELPER_USER_DIR_SUFFIX] = saved_suffix
181         return result
182
183     def operating_system(self):
184         return 'mac'
185
186     # Belongs on a Platform object.
187     def is_mavericks(self):
188         return self._version == 'mavericks'
189
190     def default_child_processes(self, **kwargs):
191         default_count = super(MacPort, self).default_child_processes()
192
193         # FIXME: https://bugs.webkit.org/show_bug.cgi?id=95906  With too many WebProcess WK2 tests get stuck in resource contention.
194         # To alleviate the issue reduce the number of running processes
195         # Anecdotal evidence suggests that a 4 core/8 core logical machine may run into this, but that a 2 core/4 core logical machine does not.
196         should_throttle_for_wk2 = self.get_option('webkit_test_runner') and default_count > 4
197         # We also want to throttle for leaks bots.
198         if should_throttle_for_wk2 or self.get_option('leaks'):
199             default_count = int(.75 * default_count)
200
201         if should_throttle_for_wk2 and self.get_option('guard_malloc'):
202             # Some 12 core Macs get a lot of tests time out when running 18 WebKitTestRunner processes (it's not clear what this depends on).
203             # <rdar://problem/25750302>
204             default_count = min(default_count, 12)
205
206         # Make sure we have enough ram to support that many instances:
207         total_memory = self.host.platform.total_bytes_memory()
208         if total_memory:
209             bytes_per_drt = 256 * 1024 * 1024  # Assume each DRT needs 256MB to run.
210             overhead = 2048 * 1024 * 1024  # Assume we need 2GB free for the O/S
211             supportable_instances = max((total_memory - overhead) / bytes_per_drt, 1)  # Always use one process, even if we don't have space for it.
212             if supportable_instances < default_count:
213                 _log.warning("This machine could support %s child processes, but only has enough memory for %s." % (default_count, supportable_instances))
214         else:
215             _log.warning("Cannot determine available memory for child processes, using default child process count of %s." % default_count)
216             supportable_instances = default_count
217         return min(supportable_instances, default_count)
218
219     def _build_java_test_support(self):
220         # FIXME: This is unused. Remove.
221         java_tests_path = self._filesystem.join(self.layout_tests_dir(), "java")
222         build_java = [self.make_command(), "-C", java_tests_path]
223         if self._executive.run_command(build_java, return_exit_code=True):  # Paths are absolute, so we don't need to set a cwd.
224             _log.error("Failed to build Java support files: %s" % build_java)
225             return False
226         return True
227
228     def _check_port_build(self):
229         return not self.get_option('java') or self._build_java_test_support()
230
231     def start_helper(self, pixel_tests=False):
232         helper_path = self._path_to_helper()
233         if not helper_path:
234             _log.error("No path to LayoutTestHelper binary")
235             return False
236         _log.debug("Starting layout helper %s" % helper_path)
237         arguments = [helper_path, '--install-color-profile']
238         self._helper = self._executive.popen(arguments,
239             stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
240         is_ready = self._helper.stdout.readline()
241         if not is_ready.startswith('ready'):
242             _log.error("LayoutTestHelper could not start")
243             return False
244         return True
245
246     def reset_preferences(self):
247         _log.debug("Resetting persistent preferences")
248
249         for domain in ["DumpRenderTree", "WebKitTestRunner"]:
250             try:
251                 self._executive.run_command(["defaults", "delete", domain])
252             except ScriptError as e:
253                 # 'defaults' returns 1 if the domain did not exist
254                 if e.exit_code != 1:
255                     raise e
256
257     def stop_helper(self):
258         if self._helper:
259             _log.debug("Stopping LayoutTestHelper")
260             try:
261                 self._helper.stdin.write("x\n")
262                 self._helper.stdin.close()
263                 self._helper.wait()
264             except IOError as e:
265                 _log.debug("IOError raised while stopping helper: %s" % str(e))
266             self._helper = None
267
268     def logging_patterns_to_strip(self):
269         # FIXME: Remove this after <rdar://problem/15605007> and <rdar://problem/35954459> are fixed.
270         return [(re.compile('(AVF|GVA) info:.*\n'), ''), ('AVDCreateGPUAccelerator: Error loading GPU renderer\n', '')]
271
272     def stderr_patterns_to_strip(self):
273         worthless_patterns = []
274         worthless_patterns.append((re.compile('.*(Fig|fig|itemasync|vt|mv_|PullParamSetSPS|ccrp_|client).* signalled err=.*\n'), ''))
275         worthless_patterns.append((re.compile('.*<<<< FigFilePlayer >>>>.*\n'), ''))
276         worthless_patterns.append((re.compile('.*<<<< FigFile >>>>.*\n'), ''))
277         worthless_patterns.append((re.compile('.*<<<< FAQ >>>>.*\n'), ''))
278         worthless_patterns.append((re.compile('.*<<<< MediaValidator >>>>.*\n'), ''))
279         worthless_patterns.append((re.compile('.*<<<< VMC >>>>.*\n'), ''))
280         worthless_patterns.append((re.compile('.*<<< FFR_Common >>>.*\n'), ''))
281         return worthless_patterns
282
283     def configuration_for_upload(self, host=None):
284         host = host or self.host
285         configuration = super(MacPort, self).configuration_for_upload(host=host)
286
287         output = host.executive.run_command(['/usr/sbin/sysctl', 'hw.model']).rstrip()
288         match = re.match(r'hw.model: (?P<model>.*)', output)
289         if match:
290             configuration['model'] = match.group('model')
291
292         return configuration