Allow a port to run tests with a custom device setup
[WebKit-https.git] / Tools / Scripts / webkitpy / port / driver.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2015, 2016 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import base64
31 import copy
32 import logging
33 import re
34 import shlex
35 import sys
36 import time
37 import os
38
39 from webkitpy.common.system import path
40 from webkitpy.common.system.profiler import ProfilerFactory
41 from webkitpy.layout_tests.servers.web_platform_test_server import WebPlatformTestServer
42
43
44 _log = logging.getLogger(__name__)
45
46
47 class DriverInput(object):
48     def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, args=None):
49         self.test_name = test_name
50         self.timeout = timeout  # in ms
51         self.image_hash = image_hash
52         self.should_run_pixel_test = should_run_pixel_test
53         self.args = args or []
54
55     def __repr__(self):
56         return "DriverInput(test_name='{}', timeout={}, image_hash={}, should_run_pixel_test={}'".format(self.test_name, self.timeout, self.image_hash, self.should_run_pixel_test)
57
58
59 class DriverOutput(object):
60     """Groups information about a output from driver for easy passing
61     and post-processing of data."""
62
63     metrics_patterns = []
64     metrics_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
65     metrics_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
66     metrics_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
67     metrics_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
68     metrics_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
69     metrics_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
70     metrics_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
71     metrics_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
72     metrics_patterns.append((re.compile('\([0-9]+px'), 'px'))
73     metrics_patterns.append((re.compile(' *" *\n +" *'), ' '))
74     metrics_patterns.append((re.compile('" +$'), '"'))
75     metrics_patterns.append((re.compile('- '), '-'))
76     metrics_patterns.append((re.compile('\n( *)"\s+'), '\n\g<1>"'))
77     metrics_patterns.append((re.compile('\s+"\n'), '"\n'))
78     metrics_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
79     metrics_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))
80     metrics_patterns.append((re.compile('scrollX [0-9]+'), 'scrollX'))
81     metrics_patterns.append((re.compile('scrollY [0-9]+'), 'scrollY'))
82     metrics_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled'))
83
84     def __init__(self, text, image, image_hash, audio, crash=False,
85             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
86             crashed_pid=None, crash_log=None, pid=None):
87         # FIXME: Args could be renamed to better clarify what they do.
88         self.text = text
89         self.image = image  # May be empty-string if the test crashes.
90         self.image_hash = image_hash
91         self.image_diff = None  # image_diff gets filled in after construction.
92         self.audio = audio  # Binary format is port-dependent.
93         self.crash = crash
94         self.crashed_process_name = crashed_process_name
95         self.crashed_pid = crashed_pid
96         self.crash_log = crash_log
97         self.test_time = test_time
98         self.measurements = measurements
99         self.timeout = timeout
100         self.error = error  # stderr output
101         self.pid = pid
102
103     def has_stderr(self):
104         return bool(self.error)
105
106     def strip_metrics(self):
107         self.strip_patterns(self.metrics_patterns)
108
109     def strip_patterns(self, patterns):
110         if not self.text:
111             return
112         for pattern in patterns:
113             self.text = re.sub(pattern[0], pattern[1], self.text)
114
115     def strip_stderror_patterns(self, patterns):
116         if not self.error:
117             return
118         for pattern in patterns:
119             self.error = re.sub(pattern[0], pattern[1], self.error)
120
121
122 class Driver(object):
123     """object for running test(s) using DumpRenderTree/WebKitTestRunner."""
124
125     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
126         """Initialize a Driver to subsequently run tests.
127
128         Typically this routine will spawn DumpRenderTree in a config
129         ready for subsequent input.
130
131         port - reference back to the port object.
132         worker_number - identifier for a particular worker/driver instance
133         """
134         self._port = port
135         self._worker_number = worker_number
136         self._no_timeout = no_timeout
137
138         self._driver_tempdir = None
139         self._driver_user_directory_suffix = None
140         self._driver_user_cache_directory = None
141
142         # WebKitTestRunner/LayoutTestRelay can report back subprocess crashes by printing
143         # "#CRASHED - PROCESSNAME".  Since those can happen at any time and ServerProcess
144         # won't be aware of them (since the actual tool didn't crash, just a subprocess)
145         # we record the crashed subprocess name here.
146         self._crashed_process_name = None
147         self._crashed_pid = None
148
149         self._driver_timed_out = False
150
151         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
152         # stderr output, as well as if we've seen #EOF on this driver instance.
153         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
154         # instead scope these locally in run_test.
155         self.error_from_test = str()
156         self.err_seen_eof = False
157         self._server_process = None
158
159         self._measurements = {}
160         if self._port.get_option("profile"):
161             profiler_name = self._port.get_option("profiler")
162             self._profiler = ProfilerFactory.create_profiler(self._port.host,
163                 self._port._path_to_driver(), self._port.results_directory(), profiler_name)
164         else:
165             self._profiler = None
166
167         self.web_platform_test_server_doc_root = self._port.web_platform_test_server_doc_root()
168         self.web_platform_test_server_base_url = self._port.web_platform_test_server_base_url()
169
170     def __del__(self):
171         self.stop()
172
173     def run_test(self, driver_input, stop_when_done):
174         """Run a single test and return the results.
175
176         Note that it is okay if a test times out or crashes and leaves
177         the driver in an indeterminate state. The upper layers of the program
178         are responsible for cleaning up and ensuring things are okay.
179
180         Returns a DriverOutput object.
181         """
182         start_time = time.time()
183         self.start(driver_input.should_run_pixel_test, driver_input.args)
184         test_begin_time = time.time()
185         self._driver_timed_out = False
186         self._crash_report_from_driver = None
187         self.error_from_test = str()
188         self.err_seen_eof = False
189
190         command = self._command_from_driver_input(driver_input)
191
192         # Certain timeouts are detected by the tool itself; tool detection is better,
193         # because results contain partial output in this case. Make script timeout longer
194         # by 5 seconds to avoid racing for which timeout is detected first.
195         # FIXME: It's not the job of the driver to decide what the timeouts should be.
196         # Move the additional timeout to driver_input.
197         if self._no_timeout:
198             deadline = test_begin_time + 60 * 60 * 24 * 7  # 7 days. Using sys.maxint causes a hang.
199         else:
200             deadline = test_begin_time + int(driver_input.timeout) / 1000.0 + 5
201
202         self._server_process.write(command)
203         text, audio = self._read_first_block(deadline, driver_input.test_name)  # First block is either text or audio
204         image, actual_image_hash = self._read_optional_image_block(deadline, driver_input.test_name)  # The second (optional) block is image data.
205
206         crashed = self.has_crashed()
207         timed_out = self._server_process.timed_out
208         driver_timed_out = self._driver_timed_out
209         pid = self._server_process.pid()
210
211         if stop_when_done or crashed or timed_out:
212             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
213             # In the timeout case, we kill the hung process as well.
214             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
215             if out:
216                 text += out
217             if err:
218                 self.error_from_test += err
219             self._server_process = None
220
221         crash_log = None
222         if self._crash_report_from_driver:
223             crash_log = self._crash_report_from_driver
224         elif crashed:
225             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
226             # If we don't find a crash log use a placeholder error message instead.
227             if not crash_log:
228                 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid"
229                 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str)
230
231                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
232                 if self.error_from_test:
233                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
234
235         return DriverOutput(text, image, actual_image_hash, audio,
236             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
237             timeout=timed_out or driver_timed_out, error=self.error_from_test,
238             crashed_process_name=self._crashed_process_name,
239             crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
240
241     def _get_crash_log(self, stdout, stderr, newer_than):
242         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)
243
244     def _command_wrapper(self):
245         # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
246         wrapper_arguments = []
247         if self._profiler:
248             wrapper_arguments = self._profiler.wrapper_arguments()
249         if self._port.get_option('wrapper'):
250             return shlex.split(self._port.get_option('wrapper')) + wrapper_arguments
251         return wrapper_arguments
252
253     HTTP_DIR = "http/tests/"
254     HTTP_LOCAL_DIR = "http/tests/local/"
255
256     def is_http_test(self, test_name):
257         return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)
258
259     def is_web_platform_test(self, test_name):
260         return test_name.startswith(self.web_platform_test_server_doc_root)
261
262     def test_to_uri(self, test_name):
263         """Convert a test name to a URI."""
264         if self.is_web_platform_test(test_name):
265             return self.web_platform_test_server_base_url + test_name[len(self.web_platform_test_server_doc_root):]
266
267         if not self.is_http_test(test_name):
268             return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))
269
270         relative_path = test_name[len(self.HTTP_DIR):]
271
272         # TODO(dpranke): remove the SSL reference?
273         if relative_path.startswith("ssl/"):
274             return "https://127.0.0.1:8443/" + relative_path
275         return "http://127.0.0.1:8000/" + relative_path
276
277     def uri_to_test(self, uri):
278         """Return the base layout test name for a given URI.
279
280         This returns the test name for a given URI, e.g., if you passed in
281         "file:///src/LayoutTests/fast/html/keygen.html" it would return
282         "fast/html/keygen.html".
283
284         """
285         if uri.startswith("file:///"):
286             prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
287             if not prefix.endswith('/'):
288                 prefix += '/'
289             return uri[len(prefix):]
290         if uri.startswith(self.web_platform_test_server_base_url):
291             return uri.replace(self.web_platform_test_server_base_url, self.web_platform_test_server_doc_root)
292         if uri.startswith("http://"):
293             return uri.replace('http://127.0.0.1:8000/', self.HTTP_DIR)
294         if uri.startswith("https://"):
295             return uri.replace('https://127.0.0.1:8443/', self.HTTP_DIR)
296         raise NotImplementedError('unknown url type: %s' % uri)
297
298     def has_crashed(self):
299         if self._server_process is None:
300             return False
301         if self._crashed_process_name:
302             return True
303         if self._server_process.has_crashed():
304             self._crashed_process_name = self._server_process.name()
305             self._crashed_pid = self._server_process.pid()
306             return True
307         return False
308
309     def start(self, pixel_tests, per_test_args):
310         # FIXME: Callers shouldn't normally call this, since this routine
311         # may not be specifying the correct combination of pixel test and
312         # per_test args.
313         #
314         # The only reason we have this routine at all is so the perftestrunner
315         # can pause before running a test; it might be better to push that
316         # into run_test() directly.
317         if not self._server_process:
318             self._start(pixel_tests, per_test_args)
319             self._run_post_start_tasks()
320
321     def _append_environment_variable_path(self, environment, variable, path):
322         if variable in environment:
323             environment[variable] = environment[variable] + os.pathsep + path
324         else:
325             environment[variable] = path
326
327     def _setup_environ_for_driver(self, environment):
328         build_root_path = str(self._port._build_path())
329         # FIXME: DYLD_* variables should be Mac-only. Even iOS Simulator doesn't need them, as LayoutTestRelay is a host binary.
330         self._append_environment_variable_path(environment, 'DYLD_LIBRARY_PATH', build_root_path)
331         self._append_environment_variable_path(environment, '__XPC_DYLD_LIBRARY_PATH', build_root_path)
332         self._append_environment_variable_path(environment, 'DYLD_FRAMEWORK_PATH', build_root_path)
333         self._append_environment_variable_path(environment, '__XPC_DYLD_FRAMEWORK_PATH', build_root_path)
334         # Use an isolated temp directory that can be deleted after testing (especially important on Mac, as
335         # CoreMedia disk cache is in the temp directory).
336         environment['TMPDIR'] = str(self._driver_tempdir)
337         environment['DIRHELPER_USER_DIR_SUFFIX'] = self._driver_user_directory_suffix
338         # Put certain normally persistent files into the temp directory (e.g. IndexedDB storage).
339         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
340         environment['LOCAL_RESOURCE_ROOT'] = str(self._port.layout_tests_dir())
341         environment['ASAN_OPTIONS'] = "allocator_may_return_null=1"
342         environment['__XPC_ASAN_OPTIONS'] = environment['ASAN_OPTIONS']
343         if 'WEBKIT_OUTPUTDIR' in os.environ:
344             environment['WEBKIT_OUTPUTDIR'] = os.environ['WEBKIT_OUTPUTDIR']
345         if self._profiler:
346             environment = self._profiler.adjusted_environment(environment)
347         return environment
348
349     def _start(self, pixel_tests, per_test_args):
350         self.stop()
351         # Each driver process should be using individual directories under _driver_tempdir (which is deleted when stopping),
352         # however some subsystems on some platforms could end up using process default ones.
353         self._port._clear_global_caches_and_temporary_files()
354         self._driver_tempdir = self._port._driver_tempdir()
355         self._driver_user_directory_suffix = os.path.basename(str(self._driver_tempdir))
356         user_cache_directory = self._port._path_to_user_cache_directory(self._driver_user_directory_suffix)
357         if user_cache_directory:
358             self._port._filesystem.maybe_make_directory(user_cache_directory)
359             self._driver_user_cache_directory = user_cache_directory
360         server_name = self._port.driver_name()
361         environment = self._port.setup_environ_for_server(server_name)
362         environment = self._setup_environ_for_driver(environment)
363         self._crashed_process_name = None
364         self._crashed_pid = None
365         self._server_process = self._port._server_process_constructor(self._port, server_name, self.cmd_line(pixel_tests, per_test_args), environment)
366         self._server_process.start()
367
368     def _run_post_start_tasks(self):
369         # Remote drivers may override this to delay post-start tasks until the server has ack'd.
370         if self._profiler:
371             self._profiler.attach_to_pid(self._pid_on_target())
372
373     def _pid_on_target(self):
374         # Remote drivers will override this method to return the pid on the device.
375         return self._server_process.pid()
376
377     def stop(self):
378         if self._server_process:
379             self._server_process.stop(self._port.driver_stop_timeout())
380             self._server_process = None
381             if self._profiler:
382                 self._profiler.profile_after_exit()
383
384         if self._driver_tempdir:
385             self._port._filesystem.rmtree(str(self._driver_tempdir))
386             self._driver_tempdir = None
387         if self._driver_user_cache_directory:
388             self._port._filesystem.rmtree(self._driver_user_cache_directory)
389             self._driver_user_cache_directory = None
390
391     def cmd_line(self, pixel_tests, per_test_args):
392         cmd = self._command_wrapper()
393         cmd.append(self._port._path_to_driver())
394         if self._port.get_option('gc_between_tests'):
395             cmd.append('--gc-between-tests')
396         if self._port.get_option('complex_text'):
397             cmd.append('--complex-text')
398         if self._port.get_option('accelerated_drawing'):
399             cmd.append('--accelerated-drawing')
400         if self._port.get_option('remote_layer_tree'):
401             cmd.append('--remote-layer-tree')
402         if self._port.get_option('threaded'):
403             cmd.append('--threaded')
404         if self._no_timeout:
405             cmd.append('--no-timeout')
406
407         for allowed_host in self._port.allowed_hosts():
408             cmd.append('--allowed-host')
409             cmd.append(allowed_host)
410
411         cmd.extend(self._port.get_option('additional_drt_flag', []))
412         cmd.extend(self._port.additional_drt_flag())
413
414         cmd.extend(per_test_args)
415
416         cmd.append('-')
417         return cmd
418
419     def _check_for_driver_timeout(self, out_line):
420         if out_line == "FAIL: Timed out waiting for notifyDone to be called\n":
421             self._driver_timed_out = True
422
423     def _check_for_address_sanitizer_violation(self, error_line):
424         if "ERROR: AddressSanitizer" in error_line:
425             return True
426
427     def _check_for_driver_crash_or_unresponsiveness(self, error_line):
428         crashed_check = error_line.rstrip('\r\n')
429         if crashed_check == "#CRASHED":
430             self._crashed_process_name = self._server_process.name()
431             self._crashed_pid = self._server_process.pid()
432             return True
433         elif error_line.startswith("#CRASHED - "):
434             match = re.match('#CRASHED - (\S+)', error_line)
435             self._crashed_process_name = match.group(1) if match else 'WebProcess'
436             match = re.search('pid (\d+)', error_line)
437             self._crashed_pid = int(match.group(1)) if match else None
438             _log.debug('%s crash, pid = %s' % (self._crashed_process_name, str(self._crashed_pid)))
439             return True
440         elif error_line.startswith("#PROCESS UNRESPONSIVE - "):
441             match = re.match('#PROCESS UNRESPONSIVE - (\S+)', error_line)
442             child_process_name = match.group(1) if match else 'WebProcess'
443             match = re.search('pid (\d+)', error_line)
444             child_process_pid = int(match.group(1)) if match else None
445             _log.debug('%s is unresponsive, pid = %s' % (child_process_name, str(child_process_pid)))
446             self._driver_timed_out = True
447             if child_process_pid:
448                 self._port.sample_process(child_process_name, child_process_pid)
449             self.error_from_test += error_line
450             self._server_process.write('#SAMPLE FINISHED\n')
451             return True
452         return self.has_crashed()
453
454     def _command_from_driver_input(self, driver_input):
455         # FIXME: performance tests pass in full URLs instead of test names.
456         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
457             command = driver_input.test_name
458         elif self.is_web_platform_test(driver_input.test_name) or (self.is_http_test(driver_input.test_name) and (self._port.get_option('webkit_test_runner') or sys.platform == "cygwin")):
459             command = self.test_to_uri(driver_input.test_name)
460         else:
461             command = self._port.abspath_for_test(driver_input.test_name)
462             if sys.platform == 'cygwin':
463                 command = path.cygpath(command)
464
465         assert not driver_input.image_hash or driver_input.should_run_pixel_test
466
467         # ' is the separator between arguments.
468         if self._port.supports_per_test_timeout():
469             command += "'--timeout'%s" % driver_input.timeout
470         if driver_input.should_run_pixel_test:
471             command += "'--pixel-test"
472         if driver_input.image_hash:
473             command += "'" + driver_input.image_hash
474         return command + "\n"
475
476     def _read_first_block(self, deadline, test_name):
477         # returns (text_content, audio_content)
478         block = self._read_block(deadline, test_name)
479         if block.malloc:
480             self._measurements['Malloc'] = float(block.malloc)
481         if block.js_heap:
482             self._measurements['JSHeap'] = float(block.js_heap)
483         if block.content_type == 'audio/wav':
484             return (None, block.decoded_content)
485         return (block.decoded_content, None)
486
487     def _read_optional_image_block(self, deadline, test_name):
488         # returns (image, actual_image_hash)
489         block = self._read_block(deadline, test_name, wait_for_stderr_eof=True)
490         if block.content and block.content_type == 'image/png':
491             return (block.decoded_content, block.content_hash)
492         return (None, block.content_hash)
493
494     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
495         if line.startswith(header_text) and getattr(block, header_attr) is None:
496             value = line.split()[1]
497             if header_filter:
498                 value = header_filter(value)
499             setattr(block, header_attr, value)
500             return True
501         return False
502
503     def _process_stdout_line(self, block, line):
504         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
505             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
506             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
507             or self._read_header(block, line, 'ActualHash: ', 'content_hash')
508             or self._read_header(block, line, 'DumpMalloc: ', 'malloc')
509             or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')):
510             return
511         # Note, we're not reading ExpectedHash: here, but we could.
512         # If the line wasn't a header, we just append it to the content.
513         block.content += line
514
515     def _strip_eof(self, line):
516         if line and line.endswith("#EOF\n"):
517             return line[:-5], True
518         return line, False
519
520     def _read_block(self, deadline, test_name, wait_for_stderr_eof=False):
521         block = ContentBlock()
522         out_seen_eof = False
523         asan_violation_detected = False
524
525         while not self.has_crashed():
526             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
527                 break
528
529             if self.err_seen_eof:
530                 out_line = self._server_process.read_stdout_line(deadline)
531                 err_line = None
532             elif out_seen_eof:
533                 out_line = None
534                 err_line = self._server_process.read_stderr_line(deadline)
535             else:
536                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
537
538             if self._server_process.timed_out or self.has_crashed():
539                 break
540
541             if out_line:
542                 assert not out_seen_eof
543                 out_line, out_seen_eof = self._strip_eof(out_line)
544             if err_line:
545                 assert not self.err_seen_eof
546                 err_line, self.err_seen_eof = self._strip_eof(err_line)
547
548             if out_line:
549                 self._check_for_driver_timeout(out_line)
550                 if out_line[-1] != "\n":
551                     _log.error("  %s -> Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug." % test_name)
552                 content_length_before_header_check = block._content_length
553                 self._process_stdout_line(block, out_line)
554                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
555                 # Don't wait until we're done with headers, just read the binary blob right now.
556                 if content_length_before_header_check != block._content_length:
557                     block.content = self._server_process.read_stdout(deadline, block._content_length)
558
559             if err_line:
560                 if self._check_for_driver_crash_or_unresponsiveness(err_line):
561                     break
562                 elif self._check_for_address_sanitizer_violation(err_line):
563                     asan_violation_detected = True
564                     self._crash_report_from_driver = ""
565                     # ASan report starts with a nondescript line, we only detect the second line.
566                     end_of_previous_error_line = self.error_from_test.rfind('\n', 0, -1)
567                     if end_of_previous_error_line > 0:
568                         self.error_from_test = self.error_from_test[:end_of_previous_error_line]
569                     else:
570                         self.error_from_test = ""
571                     # Symbolication can take a very long time, give it 10 extra minutes to finish.
572                     # FIXME: This can likely be removed once <rdar://problem/18701447> is fixed.
573                     deadline += 10 * 60 * 1000
574                 if asan_violation_detected:
575                     self._crash_report_from_driver += err_line
576                 else:
577                     self.error_from_test += err_line
578
579         if asan_violation_detected and not self._crashed_process_name:
580             self._crashed_process_name = self._server_process.name()
581             self._crashed_pid = self._server_process.pid()
582
583         block.decode_content()
584         return block
585
586     @staticmethod
587     def check_driver(port):
588         # This checks if the required system dependencies for the driver are met.
589         # Since this is the generic class implementation, just return True.
590         return True
591
592
593 # FIXME: this should be abstracted out via the Port subclass somehow.
594 class IOSSimulatorDriver(Driver):
595     def cmd_line(self, pixel_tests, per_test_args):
596         cmd = super(IOSSimulatorDriver, self).cmd_line(pixel_tests, per_test_args)
597         relay_tool = self._port.relay_path
598         dump_tool = cmd[0]
599         dump_tool_args = cmd[1:]
600         product_dir = self._port._build_path()
601         relay_args = [
602             '-udid', self._port.testing_device(self._worker_number).udid,
603             '-productDir', product_dir,
604             '-app', dump_tool,
605         ]
606         return [relay_tool] + relay_args + ['--'] + dump_tool_args
607
608     def _setup_environ_for_driver(self, environment):
609         environment['DEVELOPER_DIR'] = self._port.developer_dir
610         return super(IOSSimulatorDriver, self)._setup_environ_for_driver(environment)
611
612
613 class ContentBlock(object):
614     def __init__(self):
615         self.content_type = None
616         self.encoding = None
617         self.content_hash = None
618         self._content_length = None
619         # Content is treated as binary data even though the text output is usually UTF-8.
620         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
621         self.decoded_content = None
622         self.malloc = None
623         self.js_heap = None
624
625     def decode_content(self):
626         if self.encoding == 'base64' and self.content is not None:
627             self.decoded_content = base64.b64decode(self.content)
628         else:
629             self.decoded_content = self.content
630
631
632 class DriverProxy(object):
633     """A wrapper for managing two Driver instances, one with pixel tests and
634     one without. This allows us to handle plain text tests and ref tests with a
635     single driver."""
636
637     def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
638         self._port = port
639         self._worker_number = worker_number
640         self._driver_instance_constructor = driver_instance_constructor
641         self._no_timeout = no_timeout
642
643         # FIXME: We shouldn't need to create a driver until we actually run a test.
644         self._driver = self._make_driver(pixel_tests)
645         self._driver_cmd_line = None
646
647     def _make_driver(self, pixel_tests):
648         return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)
649
650     # FIXME: this should be a @classmethod (or implemented on Port instead).
651     def is_http_test(self, test_name):
652         return self._driver.is_http_test(test_name)
653
654     # FIXME: this should be a @classmethod (or implemented on Port instead).
655     def test_to_uri(self, test_name):
656         return self._driver.test_to_uri(test_name)
657
658     # FIXME: this should be a @classmethod (or implemented on Port instead).
659     def uri_to_test(self, uri):
660         return self._driver.uri_to_test(uri)
661
662     def run_test(self, driver_input, stop_when_done):
663         pixel_tests_needed = driver_input.should_run_pixel_test
664         cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
665         if cmd_line_key != self._driver_cmd_line:
666             self._driver.stop()
667             self._driver = self._make_driver(pixel_tests_needed)
668             self._driver_cmd_line = cmd_line_key
669
670         return self._driver.run_test(driver_input, stop_when_done)
671
672     def has_crashed(self):
673         return self._driver.has_crashed()
674
675     def stop(self):
676         self._driver.stop()
677
678     # FIXME: this should be a @classmethod (or implemented on Port instead).
679     def cmd_line(self, pixel_tests=None, per_test_args=None):
680         return self._driver.cmd_line(pixel_tests, per_test_args or [])
681
682     def _cmd_line_as_key(self, pixel_tests, per_test_args):
683         return ' '.join(self.cmd_line(pixel_tests, per_test_args))