11e8015a9a153a9e053ae2d9a360af7cfa671f36
[WebKit-https.git] / Tools / Scripts / webkitpy / port / driver.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2015, 2016 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import base64
31 import logging
32 import re
33 import shlex
34 import sys
35 import time
36 import os
37 from collections import defaultdict
38
39 from os.path import normpath
40 from webkitpy.common.system import path
41 from webkitpy.common.system.profiler import ProfilerFactory
42
43
44 _log = logging.getLogger(__name__)
45
46
47 class DriverInput(object):
48     def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, should_dump_jsconsolelog_in_stderr=None, args=None):
49         self.test_name = test_name
50         self.timeout = timeout  # in ms
51         self.image_hash = image_hash
52         self.should_run_pixel_test = should_run_pixel_test
53         self.should_dump_jsconsolelog_in_stderr = should_dump_jsconsolelog_in_stderr
54         self.args = args or []
55
56     def __repr__(self):
57         return "DriverInput(test_name='{}', timeout={}, image_hash={}, should_run_pixel_test={}, should_dump_jsconsolelog_in_stderr={}'".format(self.test_name, self.timeout, self.image_hash, self.should_run_pixel_test, self.should_dump_jsconsolelog_in_stderr)
58
59
60 class DriverOutput(object):
61     """Groups information about a output from driver for easy passing
62     and post-processing of data."""
63
64     metrics_patterns = []
65     metrics_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
66     metrics_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
67     metrics_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
68     metrics_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
69     metrics_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
70     metrics_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
71     metrics_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
72     metrics_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
73     metrics_patterns.append((re.compile('\([0-9]+px'), 'px'))
74     metrics_patterns.append((re.compile(' *" *\n +" *'), ' '))
75     metrics_patterns.append((re.compile('" +$'), '"'))
76     metrics_patterns.append((re.compile('- '), '-'))
77     metrics_patterns.append((re.compile('\n( *)"\s+'), '\n\g<1>"'))
78     metrics_patterns.append((re.compile('\s+"\n'), '"\n'))
79     metrics_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
80     metrics_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))
81     metrics_patterns.append((re.compile('scrollX [0-9]+'), 'scrollX'))
82     metrics_patterns.append((re.compile('scrollY [0-9]+'), 'scrollY'))
83     metrics_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled'))
84
85     def __init__(self, text, image, image_hash, audio, crash=False,
86             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
87             crashed_pid=None, crash_log=None, pid=None):
88         # FIXME: Args could be renamed to better clarify what they do.
89         self.text = text
90         self.image = image  # May be empty-string if the test crashes.
91         self.image_hash = image_hash
92         self.image_diff = None  # image_diff gets filled in after construction.
93         self.audio = audio  # Binary format is port-dependent.
94         self.crash = crash
95         self.crashed_process_name = crashed_process_name
96         self.crashed_pid = crashed_pid
97         self.crash_log = crash_log
98         self.test_time = test_time
99         self.measurements = measurements
100         self.timeout = timeout
101         self.error = error  # stderr output
102         self.pid = pid
103
104     def has_stderr(self):
105         return bool(self.error)
106
107     def strip_metrics(self):
108         self.strip_patterns(self.metrics_patterns)
109
110     def strip_patterns(self, patterns):
111         if not self.text:
112             return
113         for pattern in patterns:
114             self.text = re.sub(pattern[0], pattern[1], self.text)
115
116     def strip_stderror_patterns(self, patterns):
117         if not self.error:
118             return
119         for pattern in patterns:
120             self.error = re.sub(pattern[0], pattern[1], self.error)
121
122
123 class DriverPostTestOutput(object):
124     """Groups data collected for a set of tests, collected after all those testse have run
125     (for example, data about leaked objects)"""
126     def __init__(self, world_leaks_dict):
127         self.world_leaks_dict = world_leaks_dict
128
129
130 class Driver(object):
131     """object for running test(s) using DumpRenderTree/WebKitTestRunner."""
132
133     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
134         """Initialize a Driver to subsequently run tests.
135
136         Typically this routine will spawn DumpRenderTree in a config
137         ready for subsequent input.
138
139         port - reference back to the port object.
140         worker_number - identifier for a particular worker/driver instance
141         """
142         self._port = port
143         self._worker_number = worker_number
144         self._no_timeout = no_timeout
145         self._target_host = port.target_host(worker_number)
146
147         self._driver_tempdir = None
148         self._driver_user_directory_suffix = None
149         self._driver_user_cache_directory = None
150
151         # WebKitTestRunner can report back subprocess crashes by printing
152         # "#CRASHED - PROCESSNAME".  Since those can happen at any time and ServerProcess
153         # won't be aware of them (since the actual tool didn't crash, just a subprocess)
154         # we record the crashed subprocess name here.
155         self._crashed_process_name = None
156         self._crashed_pid = None
157
158         self._driver_timed_out = False
159
160         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
161         # stderr output, as well as if we've seen #EOF on this driver instance.
162         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
163         # instead scope these locally in run_test.
164         self.error_from_test = str()
165         self.err_seen_eof = False
166
167         self._server_name = self._port.driver_name()
168         self._server_process = None
169
170         self._measurements = {}
171         if self._port.get_option("profile"):
172             profiler_name = self._port.get_option("profiler")
173             self._profiler = ProfilerFactory.create_profiler(self._port.host,
174                 self._port._path_to_driver(), self._port.results_directory(), profiler_name)
175         else:
176             self._profiler = None
177
178         self.web_platform_test_server_doc_root = self._port.web_platform_test_server_doc_root()
179         self.web_platform_test_server_base_http_url = self._port.web_platform_test_server_base_http_url()
180         self.web_platform_test_server_base_https_url = self._port.web_platform_test_server_base_https_url()
181
182     def __del__(self):
183         self.stop()
184
185     def run_test(self, driver_input, stop_when_done):
186         """Run a single test and return the results.
187
188         Note that it is okay if a test times out or crashes and leaves
189         the driver in an indeterminate state. The upper layers of the program
190         are responsible for cleaning up and ensuring things are okay.
191
192         Returns a DriverOutput object.
193         """
194         start_time = time.time()
195         self.start(driver_input.should_run_pixel_test, driver_input.args)
196         test_begin_time = time.time()
197         self._driver_timed_out = False
198         self._crash_report_from_driver = None
199         self.error_from_test = str()
200         self.err_seen_eof = False
201
202         command = self._command_from_driver_input(driver_input)
203
204         # Certain timeouts are detected by the tool itself; tool detection is better,
205         # because results contain partial output in this case. Make script timeout longer
206         # by 5 seconds to avoid racing for which timeout is detected first.
207         # FIXME: It's not the job of the driver to decide what the timeouts should be.
208         # Move the additional timeout to driver_input.
209         if self._no_timeout:
210             deadline = test_begin_time + 60 * 60 * 24 * 7  # 7 days. Using sys.maxint causes a hang.
211         else:
212             deadline = test_begin_time + int(driver_input.timeout) / 1000.0 + 5
213
214         self._server_process.write(command)
215         text, audio = self._read_first_block(deadline, driver_input.test_name)  # First block is either text or audio
216         image, actual_image_hash = self._read_optional_image_block(deadline, driver_input.test_name)  # The second (optional) block is image data.
217
218         crashed = self.has_crashed()
219         timed_out = self._server_process.timed_out
220         driver_timed_out = self._driver_timed_out
221         pid = self._server_process.pid()
222
223         if stop_when_done or crashed or timed_out:
224             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
225             # In the timeout case, we kill the hung process as well.
226             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
227             if out:
228                 text += out
229             if err:
230                 self.error_from_test += err
231             self._server_process = None
232
233         crash_log = None
234         if self._crash_report_from_driver:
235             crash_log = self._crash_report_from_driver
236         elif crashed:
237             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
238             # If we don't find a crash log use a placeholder error message instead.
239             if not crash_log:
240                 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid"
241                 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str)
242
243                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
244                 if self.error_from_test:
245                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
246
247         return DriverOutput(text, image, actual_image_hash, audio,
248             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
249             timeout=timed_out or driver_timed_out, error=self.error_from_test,
250             crashed_process_name=self._crashed_process_name,
251             crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
252
253     def do_post_tests_work(self):
254         if not self._port.get_option('world_leaks'):
255             return None
256
257         if not self._server_process:
258             return None
259
260         _log.debug('Checking for world leaks...')
261         self._server_process.write('#CHECK FOR WORLD LEAKS\n')
262         deadline = time.time() + 20
263         block = self._read_block(deadline, '', wait_for_stderr_eof=True)
264
265         _log.debug('World leak result: %s' % (block.decoded_content))
266
267         return self._parse_world_leaks_output(block.decoded_content)
268
269     def _parse_world_leaks_output(self, output):
270         tests_with_world_leaks = defaultdict(list)
271
272         last_test = None
273         for line in output.splitlines():
274             m = re.match('^TEST: (.+)$', line)
275             if m:
276                 last_test = self.uri_to_test(m.group(1))
277             m = re.match('^ABANDONED DOCUMENT: (.+)$', line)
278             if m:
279                 leaked_document_url = m.group(1)
280                 if last_test:
281                     tests_with_world_leaks[last_test].append(leaked_document_url)
282
283         return DriverPostTestOutput(tests_with_world_leaks)
284
285     def _get_crash_log(self, stdout, stderr, newer_than):
286         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than, target_host=self._target_host)
287
288     def _command_wrapper(self):
289         # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
290         wrapper_arguments = []
291         if self._profiler:
292             wrapper_arguments = self._profiler.wrapper_arguments()
293         if self._port.get_option('wrapper'):
294             return shlex.split(self._port.get_option('wrapper')) + wrapper_arguments
295         return wrapper_arguments
296
297     HTTP_DIR = "http/tests/"
298     HTTP_LOCAL_DIR = "http/tests/local/"
299     WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR = "http/wpt/"
300     WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE = "WebKit/"
301
302     def is_http_test(self, test_name):
303         return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)
304
305     def is_webkit_specific_web_platform_test(self, test_name):
306         return test_name.startswith(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
307
308     def is_web_platform_test(self, test_name):
309         return test_name.startswith(self.web_platform_test_server_doc_root)
310
311     def wpt_test_path_to_uri(self, path):
312         return self.web_platform_test_server_base_https_url + path if ".https." in path else self.web_platform_test_server_base_http_url + path
313
314     def http_test_path_to_uri(self, path):
315         path = path.replace(os.sep, '/')
316         return self.http_base_url(secure=self.is_secure_path(path)) + path
317
318     def is_secure_path(self, path):
319         return path.startswith("ssl") or ".https." in path
320
321     def http_base_url(self, secure=None):
322         return "%s://127.0.0.1:%d/" % (('https', 8443) if secure else ('http', 8000))
323
324     def test_to_uri(self, test_name):
325         """Convert a test name to a URI."""
326         if self.is_web_platform_test(test_name):
327             return self.wpt_test_path_to_uri(test_name[len(self.web_platform_test_server_doc_root):])
328         if self.is_webkit_specific_web_platform_test(test_name):
329             return self.wpt_test_path_to_uri(self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE + test_name[len(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR):])
330
331         if not self.is_http_test(test_name):
332             return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))
333
334         return self.http_test_path_to_uri(test_name[len(self.HTTP_DIR):])
335
336     def uri_to_test(self, uri):
337         """Return the base layout test name for a given URI.
338
339         This returns the test name for a given URI, e.g., if you passed in
340         "file:///src/LayoutTests/fast/html/keygen.html" it would return
341         "fast/html/keygen.html".
342
343         """
344         if uri.startswith("file:///"):
345             prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
346             if not prefix.endswith('/'):
347                 prefix += '/'
348             return uri[len(prefix):]
349         if uri.startswith(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
350             return uri.replace(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
351         if uri.startswith(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
352             return uri.replace(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
353         if uri.startswith(self.web_platform_test_server_base_http_url):
354             return uri.replace(self.web_platform_test_server_base_http_url, self.web_platform_test_server_doc_root)
355         if uri.startswith(self.web_platform_test_server_base_https_url):
356             return uri.replace(self.web_platform_test_server_base_https_url, self.web_platform_test_server_doc_root)
357         if uri.startswith("http://"):
358             return uri.replace(self.http_base_url(secure=False), self.HTTP_DIR)
359         if uri.startswith("https://"):
360             return uri.replace(self.http_base_url(secure=True), self.HTTP_DIR)
361         raise NotImplementedError('unknown url type: %s' % uri)
362
363     def has_crashed(self):
364         if self._server_process is None:
365             return False
366         if self._crashed_process_name:
367             return True
368         if self._server_process.has_crashed():
369             self._crashed_process_name = self._server_process.process_name()
370             self._crashed_pid = self._server_process.pid()
371             return True
372         return False
373
374     def start(self, pixel_tests, per_test_args):
375         # FIXME: Callers shouldn't normally call this, since this routine
376         # may not be specifying the correct combination of pixel test and
377         # per_test args.
378         #
379         # The only reason we have this routine at all is so the perftestrunner
380         # can pause before running a test; it might be better to push that
381         # into run_test() directly.
382         if not self._server_process:
383             self._start(pixel_tests, per_test_args)
384             self._run_post_start_tasks()
385
386     def _append_environment_variable_path(self, environment, variable, path):
387         if variable in environment:
388             environment[variable] = environment[variable] + os.pathsep + path
389         else:
390             environment[variable] = path
391
392     def _setup_environ_for_driver(self, environment):
393         build_root_path = str(self._port._build_path())
394         self._append_environment_variable_path(environment, 'DYLD_LIBRARY_PATH', build_root_path)
395         self._append_environment_variable_path(environment, '__XPC_DYLD_LIBRARY_PATH', build_root_path)
396         self._append_environment_variable_path(environment, 'DYLD_FRAMEWORK_PATH', build_root_path)
397         self._append_environment_variable_path(environment, '__XPC_DYLD_FRAMEWORK_PATH', build_root_path)
398         # Use an isolated temp directory that can be deleted after testing (especially important on Mac, as
399         # CoreMedia disk cache is in the temp directory).
400         environment['TMPDIR'] = str(self._driver_tempdir)
401         environment['DIRHELPER_USER_DIR_SUFFIX'] = self._driver_user_directory_suffix
402         # Put certain normally persistent files into the temp directory (e.g. IndexedDB storage).
403         if sys.platform == 'cygwin':
404             environment['DUMPRENDERTREE_TEMP'] = path.cygpath(str(self._driver_tempdir))
405         else:
406             environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
407         environment['LOCAL_RESOURCE_ROOT'] = str(self._port.layout_tests_dir())
408         environment['ASAN_OPTIONS'] = "allocator_may_return_null=1"
409         environment['__XPC_ASAN_OPTIONS'] = environment['ASAN_OPTIONS']
410
411         # Disable vnode-guard related simulated crashes for WKTR / DRT (rdar://problem/40674034).
412         environment['SQLITE_EXEMPT_PATH_FROM_VNODE_GUARDS'] = os.path.realpath(environment['DUMPRENDERTREE_TEMP'])
413         environment['__XPC_SQLITE_EXEMPT_PATH_FROM_VNODE_GUARDS'] = environment['SQLITE_EXEMPT_PATH_FROM_VNODE_GUARDS']
414
415         if self._profiler:
416             environment = self._profiler.adjusted_environment(environment)
417         return environment
418
419     def _setup_environ_for_test(self):
420         environment = self._port.setup_environ_for_server(self._server_name)
421         environment = self._setup_environ_for_driver(environment)
422         return environment
423
424     def _start(self, pixel_tests, per_test_args):
425         self.stop()
426         # Each driver process should be using individual directories under _driver_tempdir (which is deleted when stopping),
427         # however some subsystems on some platforms could end up using process default ones.
428         self._port._clear_global_caches_and_temporary_files()
429         self._driver_tempdir = self._port._driver_tempdir(self._target_host)
430         self._driver_user_directory_suffix = os.path.basename(str(self._driver_tempdir))
431         user_cache_directory = self._port._path_to_user_cache_directory(self._driver_user_directory_suffix)
432         if user_cache_directory:
433             self._target_host.filesystem.maybe_make_directory(user_cache_directory)
434             self._driver_user_cache_directory = user_cache_directory
435         environment = self._setup_environ_for_test()
436         self._crashed_process_name = None
437         self._crashed_pid = None
438         self._server_process = self._port._test_runner_process_constructor(self._port, self._server_name, self.cmd_line(pixel_tests, per_test_args), environment, target_host=self._target_host)
439         self._server_process.start()
440
441     def _run_post_start_tasks(self):
442         # Remote drivers may override this to delay post-start tasks until the server has ack'd.
443         if self._profiler:
444             self._profiler.attach_to_pid(self._pid_on_target())
445
446     def _pid_on_target(self):
447         # Remote drivers will override this method to return the pid on the device.
448         return self._server_process.pid()
449
450     def stop(self):
451         if self._server_process:
452             self._server_process.stop(self._port.driver_stop_timeout())
453             self._server_process = None
454             if self._profiler:
455                 self._profiler.profile_after_exit()
456
457         if self._driver_tempdir:
458             self._target_host.filesystem.rmtree(str(self._driver_tempdir))
459             self._driver_tempdir = None
460         if self._driver_user_cache_directory:
461             self._target_host.filesystem.rmtree(self._driver_user_cache_directory)
462             self._driver_user_cache_directory = None
463
464     def cmd_line(self, pixel_tests, per_test_args):
465         cmd = self._command_wrapper()
466         cmd.append(self._port._path_to_driver())
467         if self._port.get_option('gc_between_tests'):
468             cmd.append('--gc-between-tests')
469         if self._port.get_option('complex_text'):
470             cmd.append('--complex-text')
471         if self._port.get_option('accelerated_drawing'):
472             cmd.append('--accelerated-drawing')
473         if self._port.get_option('remote_layer_tree'):
474             cmd.append('--remote-layer-tree')
475         if self._port.get_option('world_leaks'):
476             cmd.append('--world-leaks')
477         if self._port.get_option('threaded'):
478             cmd.append('--threaded')
479         if self._no_timeout:
480             cmd.append('--no-timeout')
481         if self._port.get_option('show_touches'):
482             cmd.append('--show-touches')
483
484         for allowed_host in self._port.allowed_hosts():
485             cmd.append('--allowed-host')
486             cmd.append(allowed_host)
487
488         cmd.extend(self._port.get_option('additional_drt_flag', []))
489         cmd.extend(self._port.additional_drt_flag())
490
491         cmd.extend(per_test_args)
492
493         cmd.append('-')
494         return cmd
495
496     def _check_for_driver_timeout(self, out_line):
497         if out_line.startswith("#PID UNRESPONSIVE - "):
498             match = re.match('#PID UNRESPONSIVE - (\S+)', out_line)
499             child_process_name = match.group(1) if match else 'WebProcess'
500             match = re.search('pid (\d+)', out_line)
501             child_process_pid = int(match.group(1)) if match else None
502             err_line = 'Wait on notifyDone timed out, process ' + child_process_name + ' pid = ' + str(child_process_pid)
503             self.error_from_test += err_line
504             _log.debug(err_line)
505             if self._port.get_option("sample_on_timeout"):
506                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
507         if out_line == "FAIL: Timed out waiting for notifyDone to be called\n":
508             self._driver_timed_out = True
509
510     def _check_for_address_sanitizer_violation(self, error_line):
511         if "ERROR: AddressSanitizer" in error_line:
512             return True
513
514     def _check_for_driver_crash_or_unresponsiveness(self, error_line):
515         crashed_check = error_line.rstrip('\r\n')
516         if crashed_check == "#CRASHED":
517             self._crashed_process_name = self._server_process.process_name()
518             self._crashed_pid = self._server_process.pid()
519             return True
520         elif error_line.startswith("#CRASHED - "):
521             match = re.match('#CRASHED - (\S+)', error_line)
522             self._crashed_process_name = match.group(1) if match else 'WebProcess'
523             match = re.search('pid (\d+)', error_line)
524             self._crashed_pid = int(match.group(1)) if match else None
525             _log.debug('%s crash, pid = %s' % (self._crashed_process_name, str(self._crashed_pid)))
526             return True
527         elif error_line.startswith("#PROCESS UNRESPONSIVE - "):
528             match = re.match('#PROCESS UNRESPONSIVE - (\S+)', error_line)
529             child_process_name = match.group(1) if match else 'WebProcess'
530             match = re.search('pid (\d+)', error_line)
531             child_process_pid = int(match.group(1)) if match else None
532             _log.debug('%s is unresponsive, pid = %s' % (child_process_name, str(child_process_pid)))
533             self._driver_timed_out = True
534             if child_process_pid:
535                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
536             self.error_from_test += error_line
537             self._server_process.write('#SAMPLE FINISHED\n', True)  # Must be able to ignore a broken pipe here, target process may already be closed.
538             return True
539         return self.has_crashed()
540
541     def _command_from_driver_input(self, driver_input):
542         # FIXME: performance tests pass in full URLs instead of test names.
543         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
544             command = driver_input.test_name
545         elif self.is_web_platform_test(driver_input.test_name) or self.is_webkit_specific_web_platform_test(driver_input.test_name) or self.is_http_test(driver_input.test_name):
546             command = self.test_to_uri(driver_input.test_name)
547             command += "'--absolutePath'"
548             command += self._port.abspath_for_test(driver_input.test_name, self._target_host)
549         else:
550             command = self._port.abspath_for_test(driver_input.test_name, self._target_host)
551             if sys.platform == 'cygwin':
552                 command = path.cygpath(command)
553
554         assert not driver_input.image_hash or driver_input.should_run_pixel_test
555
556         # ' is the separator between arguments.
557         if self._port.supports_per_test_timeout():
558             command += "'--timeout'%s" % driver_input.timeout
559         if driver_input.should_run_pixel_test:
560             command += "'--pixel-test"
561         if driver_input.should_dump_jsconsolelog_in_stderr:
562             command += "'--dump-jsconsolelog-in-stderr"
563         if driver_input.image_hash:
564             command += "'" + driver_input.image_hash
565         return command + "\n"
566
567     def _read_first_block(self, deadline, test_name):
568         # returns (text_content, audio_content)
569         block = self._read_block(deadline, test_name)
570         if block.malloc:
571             self._measurements['Malloc'] = float(block.malloc)
572         if block.js_heap:
573             self._measurements['JSHeap'] = float(block.js_heap)
574         if block.content_type == 'audio/wav':
575             return (None, block.decoded_content)
576         return (block.decoded_content, None)
577
578     def _read_optional_image_block(self, deadline, test_name):
579         # returns (image, actual_image_hash)
580         block = self._read_block(deadline, test_name, wait_for_stderr_eof=True)
581         if block.content and block.content_type == 'image/png':
582             return (block.decoded_content, block.content_hash)
583         return (None, block.content_hash)
584
585     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
586         if line.startswith(header_text) and getattr(block, header_attr) is None:
587             value = line.split()[1]
588             if header_filter:
589                 value = header_filter(value)
590             setattr(block, header_attr, value)
591             return True
592         return False
593
594     def _process_stdout_line(self, block, line):
595         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
596             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
597             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
598             or self._read_header(block, line, 'ActualHash: ', 'content_hash')
599             or self._read_header(block, line, 'DumpMalloc: ', 'malloc')
600             or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')):
601             return
602         # Note, we're not reading ExpectedHash: here, but we could.
603         # If the line wasn't a header, we just append it to the content.
604         block.content += line
605
606     def _strip_eof(self, line):
607         if line and line.endswith("#EOF\n"):
608             return line[:-5], True
609         return line, False
610
611     def _read_block(self, deadline, test_name, wait_for_stderr_eof=False):
612         block = ContentBlock()
613         out_seen_eof = False
614         asan_violation_detected = False
615
616         while True:
617             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
618                 break
619
620             if self.err_seen_eof:
621                 out_line = self._server_process.read_stdout_line(deadline)
622                 err_line = None
623             elif out_seen_eof:
624                 out_line = None
625                 err_line = self._server_process.read_stderr_line(deadline)
626             else:
627                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
628
629             # ServerProcess returns None for time outs and crashes.
630             if out_line is None and err_line is None:
631                 break
632
633             if out_line:
634                 assert not out_seen_eof
635                 out_line, out_seen_eof = self._strip_eof(out_line)
636             if err_line:
637                 assert not self.err_seen_eof
638                 err_line, self.err_seen_eof = self._strip_eof(err_line)
639
640             if out_line:
641                 self._check_for_driver_timeout(out_line)
642                 if out_line[-1] != "\n":
643                     _log.error("  %s -> Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug." % test_name)
644                 content_length_before_header_check = block._content_length
645                 self._process_stdout_line(block, out_line)
646                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
647                 # Don't wait until we're done with headers, just read the binary blob right now.
648                 if content_length_before_header_check != block._content_length:
649                     block.content = self._server_process.read_stdout(deadline, block._content_length)
650
651             if err_line:
652                 if self._check_for_driver_crash_or_unresponsiveness(err_line):
653                     break
654                 elif self._check_for_address_sanitizer_violation(err_line):
655                     asan_violation_detected = True
656                     self._crash_report_from_driver = ""
657                     # ASan report starts with a nondescript line, we only detect the second line.
658                     end_of_previous_error_line = self.error_from_test.rfind('\n', 0, -1)
659                     if end_of_previous_error_line > 0:
660                         self.error_from_test = self.error_from_test[:end_of_previous_error_line]
661                     else:
662                         self.error_from_test = ""
663                     # Symbolication can take a very long time, give it 10 extra minutes to finish.
664                     # FIXME: This can likely be removed once <rdar://problem/18701447> is fixed.
665                     deadline += 10 * 60 * 1000
666                 if asan_violation_detected:
667                     self._crash_report_from_driver += err_line
668                 else:
669                     self.error_from_test += err_line
670
671         if asan_violation_detected and not self._crashed_process_name:
672             self._crashed_process_name = self._server_process.process_name()
673             self._crashed_pid = self._server_process.pid()
674
675         block.decode_content()
676         return block
677
678     @staticmethod
679     def check_driver(port):
680         # This checks if the required system dependencies for the driver are met.
681         # Since this is the generic class implementation, just return True.
682         return True
683
684
685 class ContentBlock(object):
686     def __init__(self):
687         self.content_type = None
688         self.encoding = None
689         self.content_hash = None
690         self._content_length = None
691         # Content is treated as binary data even though the text output is usually UTF-8.
692         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
693         self.decoded_content = None
694         self.malloc = None
695         self.js_heap = None
696
697     def decode_content(self):
698         if self.encoding == 'base64' and self.content is not None:
699             self.decoded_content = base64.b64decode(self.content)
700         else:
701             self.decoded_content = self.content
702
703
704 class DriverProxy(object):
705     """A wrapper for managing two Driver instances, one with pixel tests and
706     one without. This allows us to handle plain text tests and ref tests with a
707     single driver."""
708
709     def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
710         self._port = port
711         self._worker_number = worker_number
712         self._driver_instance_constructor = driver_instance_constructor
713         self._no_timeout = no_timeout
714
715         # FIXME: We shouldn't need to create a driver until we actually run a test.
716         self._driver = self._make_driver(pixel_tests)
717         self._driver_cmd_line = None
718
719     def _make_driver(self, pixel_tests):
720         return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)
721
722     @property
723     def host(self):
724         return self._driver._target_host
725
726     # FIXME: this should be a @classmethod (or implemented on Port instead).
727     def is_http_test(self, test_name):
728         return self._driver.is_http_test(test_name)
729
730     def is_web_platform_test(self, test_name):
731         return self._driver.is_web_platform_test(test_name)
732
733     def is_webkit_specific_web_platform_test(self, test_name):
734         return self._driver.is_webkit_specific_web_platform_test(test_name)
735
736     # FIXME: this should be a @classmethod (or implemented on Port instead).
737     def test_to_uri(self, test_name):
738         return self._driver.test_to_uri(test_name)
739
740     # FIXME: this should be a @classmethod (or implemented on Port instead).
741     def uri_to_test(self, uri):
742         return self._driver.uri_to_test(uri)
743
744     def run_test(self, driver_input, stop_when_done):
745         pixel_tests_needed = driver_input.should_run_pixel_test
746         cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
747         if cmd_line_key != self._driver_cmd_line:
748             self._driver.stop()
749             self._driver = self._make_driver(pixel_tests_needed)
750             self._driver_cmd_line = cmd_line_key
751
752         return self._driver.run_test(driver_input, stop_when_done)
753
754     def do_post_tests_work(self):
755         return self._driver.do_post_tests_work()
756
757     def has_crashed(self):
758         return self._driver.has_crashed()
759
760     def stop(self):
761         self._driver.stop()
762
763     # FIXME: this should be a @classmethod (or implemented on Port instead).
764     def cmd_line(self, pixel_tests=None, per_test_args=None):
765         return self._driver.cmd_line(pixel_tests, per_test_args or [])
766
767     def _cmd_line_as_key(self, pixel_tests, per_test_args):
768         return ' '.join(self.cmd_line(pixel_tests, per_test_args))