8d4c8d4996605fd9467520d97b72bbd3753d756d
[WebKit-https.git] / Tools / Scripts / webkitpy / port / driver.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2015, 2016 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import base64
31 import logging
32 import re
33 import shlex
34 import sys
35 import time
36 import os
37
38 from webkitpy.common.system import path
39 from webkitpy.common.system.profiler import ProfilerFactory
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class DriverInput(object):
46     def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, should_dump_jsconsolelog_in_stderr=None, args=None):
47         self.test_name = test_name
48         self.timeout = timeout  # in ms
49         self.image_hash = image_hash
50         self.should_run_pixel_test = should_run_pixel_test
51         self.should_dump_jsconsolelog_in_stderr = should_dump_jsconsolelog_in_stderr
52         self.args = args or []
53
54     def __repr__(self):
55         return "DriverInput(test_name='{}', timeout={}, image_hash={}, should_run_pixel_test={}, should_dump_jsconsolelog_in_stderr={}'".format(self.test_name, self.timeout, self.image_hash, self.should_run_pixel_test, self.should_dump_jsconsolelog_in_stderr)
56
57
58 class DriverOutput(object):
59     """Groups information about a output from driver for easy passing
60     and post-processing of data."""
61
62     metrics_patterns = []
63     metrics_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
64     metrics_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
65     metrics_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
66     metrics_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
67     metrics_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
68     metrics_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
69     metrics_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
70     metrics_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
71     metrics_patterns.append((re.compile('\([0-9]+px'), 'px'))
72     metrics_patterns.append((re.compile(' *" *\n +" *'), ' '))
73     metrics_patterns.append((re.compile('" +$'), '"'))
74     metrics_patterns.append((re.compile('- '), '-'))
75     metrics_patterns.append((re.compile('\n( *)"\s+'), '\n\g<1>"'))
76     metrics_patterns.append((re.compile('\s+"\n'), '"\n'))
77     metrics_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
78     metrics_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))
79     metrics_patterns.append((re.compile('scrollX [0-9]+'), 'scrollX'))
80     metrics_patterns.append((re.compile('scrollY [0-9]+'), 'scrollY'))
81     metrics_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled'))
82
83     def __init__(self, text, image, image_hash, audio, crash=False,
84             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
85             crashed_pid=None, crash_log=None, pid=None):
86         # FIXME: Args could be renamed to better clarify what they do.
87         self.text = text
88         self.image = image  # May be empty-string if the test crashes.
89         self.image_hash = image_hash
90         self.image_diff = None  # image_diff gets filled in after construction.
91         self.audio = audio  # Binary format is port-dependent.
92         self.crash = crash
93         self.crashed_process_name = crashed_process_name
94         self.crashed_pid = crashed_pid
95         self.crash_log = crash_log
96         self.test_time = test_time
97         self.measurements = measurements
98         self.timeout = timeout
99         self.error = error  # stderr output
100         self.pid = pid
101
102     def has_stderr(self):
103         return bool(self.error)
104
105     def strip_metrics(self):
106         self.strip_patterns(self.metrics_patterns)
107
108     def strip_patterns(self, patterns):
109         if not self.text:
110             return
111         for pattern in patterns:
112             self.text = re.sub(pattern[0], pattern[1], self.text)
113
114     def strip_stderror_patterns(self, patterns):
115         if not self.error:
116             return
117         for pattern in patterns:
118             self.error = re.sub(pattern[0], pattern[1], self.error)
119
120
121 class Driver(object):
122     """object for running test(s) using DumpRenderTree/WebKitTestRunner."""
123
124     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
125         """Initialize a Driver to subsequently run tests.
126
127         Typically this routine will spawn DumpRenderTree in a config
128         ready for subsequent input.
129
130         port - reference back to the port object.
131         worker_number - identifier for a particular worker/driver instance
132         """
133         self._port = port
134         self._worker_number = worker_number
135         self._no_timeout = no_timeout
136         self._target_host = port.target_host(worker_number)
137
138         self._driver_tempdir = None
139         self._driver_user_directory_suffix = None
140         self._driver_user_cache_directory = None
141
142         # WebKitTestRunner can report back subprocess crashes by printing
143         # "#CRASHED - PROCESSNAME".  Since those can happen at any time and ServerProcess
144         # won't be aware of them (since the actual tool didn't crash, just a subprocess)
145         # we record the crashed subprocess name here.
146         self._crashed_process_name = None
147         self._crashed_pid = None
148
149         self._driver_timed_out = False
150
151         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
152         # stderr output, as well as if we've seen #EOF on this driver instance.
153         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
154         # instead scope these locally in run_test.
155         self.error_from_test = str()
156         self.err_seen_eof = False
157
158         self._server_name = self._port.driver_name()
159         self._server_process = None
160
161         self._measurements = {}
162         if self._port.get_option("profile"):
163             profiler_name = self._port.get_option("profiler")
164             self._profiler = ProfilerFactory.create_profiler(self._port.host,
165                 self._port._path_to_driver(), self._port.results_directory(), profiler_name)
166         else:
167             self._profiler = None
168
169         self.web_platform_test_server_doc_root = self._port.web_platform_test_server_doc_root()
170         self.web_platform_test_server_base_http_url = self._port.web_platform_test_server_base_http_url()
171         self.web_platform_test_server_base_https_url = self._port.web_platform_test_server_base_https_url()
172
173     def __del__(self):
174         self.stop()
175
176     def run_test(self, driver_input, stop_when_done):
177         """Run a single test and return the results.
178
179         Note that it is okay if a test times out or crashes and leaves
180         the driver in an indeterminate state. The upper layers of the program
181         are responsible for cleaning up and ensuring things are okay.
182
183         Returns a DriverOutput object.
184         """
185         start_time = time.time()
186         self.start(driver_input.should_run_pixel_test, driver_input.args)
187         test_begin_time = time.time()
188         self._driver_timed_out = False
189         self._crash_report_from_driver = None
190         self.error_from_test = str()
191         self.err_seen_eof = False
192
193         command = self._command_from_driver_input(driver_input)
194
195         # Certain timeouts are detected by the tool itself; tool detection is better,
196         # because results contain partial output in this case. Make script timeout longer
197         # by 5 seconds to avoid racing for which timeout is detected first.
198         # FIXME: It's not the job of the driver to decide what the timeouts should be.
199         # Move the additional timeout to driver_input.
200         if self._no_timeout:
201             deadline = test_begin_time + 60 * 60 * 24 * 7  # 7 days. Using sys.maxint causes a hang.
202         else:
203             deadline = test_begin_time + int(driver_input.timeout) / 1000.0 + 5
204
205         self._server_process.write(command)
206         text, audio = self._read_first_block(deadline, driver_input.test_name)  # First block is either text or audio
207         image, actual_image_hash = self._read_optional_image_block(deadline, driver_input.test_name)  # The second (optional) block is image data.
208
209         crashed = self.has_crashed()
210         timed_out = self._server_process.timed_out
211         driver_timed_out = self._driver_timed_out
212         pid = self._server_process.pid()
213
214         if stop_when_done or crashed or timed_out:
215             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
216             # In the timeout case, we kill the hung process as well.
217             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
218             if out:
219                 text += out
220             if err:
221                 self.error_from_test += err
222             self._server_process = None
223
224         crash_log = None
225         if self._crash_report_from_driver:
226             crash_log = self._crash_report_from_driver
227         elif crashed:
228             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
229             # If we don't find a crash log use a placeholder error message instead.
230             if not crash_log:
231                 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid"
232                 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str)
233
234                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
235                 if self.error_from_test:
236                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
237
238         return DriverOutput(text, image, actual_image_hash, audio,
239             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
240             timeout=timed_out or driver_timed_out, error=self.error_from_test,
241             crashed_process_name=self._crashed_process_name,
242             crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
243
244     def _get_crash_log(self, stdout, stderr, newer_than):
245         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than, target_host=self._target_host)
246
247     def _command_wrapper(self):
248         # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
249         wrapper_arguments = []
250         if self._profiler:
251             wrapper_arguments = self._profiler.wrapper_arguments()
252         if self._port.get_option('wrapper'):
253             return shlex.split(self._port.get_option('wrapper')) + wrapper_arguments
254         return wrapper_arguments
255
256     HTTP_DIR = "http/tests/"
257     HTTP_LOCAL_DIR = "http/tests/local/"
258     WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR = "http/wpt/"
259     WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE = "WebKit/"
260
261     def is_http_test(self, test_name):
262         return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)
263
264     def is_webkit_specific_web_platform_test(self, test_name):
265         return test_name.startswith(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
266
267     def is_web_platform_test(self, test_name):
268         return test_name.startswith(self.web_platform_test_server_doc_root)
269
270     def wpt_test_path_to_uri(self, path):
271         return self.web_platform_test_server_base_https_url + path if ".https." in path else self.web_platform_test_server_base_http_url + path
272
273     def http_test_path_to_uri(self, path):
274         return "https://127.0.0.1:8443/" + path if path.startswith("ssl") or ".https." in path else "http://127.0.0.1:8000/" + path
275
276     def test_to_uri(self, test_name):
277         """Convert a test name to a URI."""
278         if self.is_web_platform_test(test_name):
279             return self.wpt_test_path_to_uri(test_name[len(self.web_platform_test_server_doc_root):])
280         if self.is_webkit_specific_web_platform_test(test_name):
281             return self.wpt_test_path_to_uri(self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE + test_name[len(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR):])
282
283         if not self.is_http_test(test_name):
284             return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))
285
286         return self.http_test_path_to_uri(test_name[len(self.HTTP_DIR):])
287
288     def uri_to_test(self, uri):
289         """Return the base layout test name for a given URI.
290
291         This returns the test name for a given URI, e.g., if you passed in
292         "file:///src/LayoutTests/fast/html/keygen.html" it would return
293         "fast/html/keygen.html".
294
295         """
296         if uri.startswith("file:///"):
297             prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
298             if not prefix.endswith('/'):
299                 prefix += '/'
300             return uri[len(prefix):]
301         if uri.startswith(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
302             return uri.replace(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
303         if uri.startswith(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
304             return uri.replace(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
305         if uri.startswith(self.web_platform_test_server_base_http_url):
306             return uri.replace(self.web_platform_test_server_base_http_url, self.web_platform_test_server_doc_root)
307         if uri.startswith(self.web_platform_test_server_base_https_url):
308             return uri.replace(self.web_platform_test_server_base_https_url, self.web_platform_test_server_doc_root)
309         if uri.startswith("http://"):
310             return uri.replace('http://127.0.0.1:8000/', self.HTTP_DIR)
311         if uri.startswith("https://"):
312             return uri.replace('https://127.0.0.1:8443/', self.HTTP_DIR)
313         raise NotImplementedError('unknown url type: %s' % uri)
314
315     def has_crashed(self):
316         if self._server_process is None:
317             return False
318         if self._crashed_process_name:
319             return True
320         if self._server_process.has_crashed():
321             self._crashed_process_name = self._server_process.process_name()
322             self._crashed_pid = self._server_process.pid()
323             return True
324         return False
325
326     def start(self, pixel_tests, per_test_args):
327         # FIXME: Callers shouldn't normally call this, since this routine
328         # may not be specifying the correct combination of pixel test and
329         # per_test args.
330         #
331         # The only reason we have this routine at all is so the perftestrunner
332         # can pause before running a test; it might be better to push that
333         # into run_test() directly.
334         if not self._server_process:
335             self._start(pixel_tests, per_test_args)
336             self._run_post_start_tasks()
337
338     def _append_environment_variable_path(self, environment, variable, path):
339         if variable in environment:
340             environment[variable] = environment[variable] + os.pathsep + path
341         else:
342             environment[variable] = path
343
344     def _setup_environ_for_driver(self, environment):
345         build_root_path = str(self._port._build_path())
346         self._append_environment_variable_path(environment, 'DYLD_LIBRARY_PATH', build_root_path)
347         self._append_environment_variable_path(environment, '__XPC_DYLD_LIBRARY_PATH', build_root_path)
348         self._append_environment_variable_path(environment, 'DYLD_FRAMEWORK_PATH', build_root_path)
349         self._append_environment_variable_path(environment, '__XPC_DYLD_FRAMEWORK_PATH', build_root_path)
350         # Use an isolated temp directory that can be deleted after testing (especially important on Mac, as
351         # CoreMedia disk cache is in the temp directory).
352         environment['TMPDIR'] = str(self._driver_tempdir)
353         environment['DIRHELPER_USER_DIR_SUFFIX'] = self._driver_user_directory_suffix
354         # Put certain normally persistent files into the temp directory (e.g. IndexedDB storage).
355         if sys.platform == 'cygwin':
356             environment['DUMPRENDERTREE_TEMP'] = path.cygpath(str(self._driver_tempdir))
357         else:
358             environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
359         environment['LOCAL_RESOURCE_ROOT'] = str(self._port.layout_tests_dir())
360         environment['ASAN_OPTIONS'] = "allocator_may_return_null=1"
361         environment['__XPC_ASAN_OPTIONS'] = environment['ASAN_OPTIONS']
362         if self._profiler:
363             environment = self._profiler.adjusted_environment(environment)
364         return environment
365
366     def _setup_environ_for_test(self):
367         environment = self._port.setup_environ_for_server(self._server_name)
368         environment = self._setup_environ_for_driver(environment)
369         return environment
370
371     def _start(self, pixel_tests, per_test_args):
372         self.stop()
373         # Each driver process should be using individual directories under _driver_tempdir (which is deleted when stopping),
374         # however some subsystems on some platforms could end up using process default ones.
375         self._port._clear_global_caches_and_temporary_files()
376         self._driver_tempdir = self._port._driver_tempdir(self._target_host)
377         self._driver_user_directory_suffix = os.path.basename(str(self._driver_tempdir))
378         user_cache_directory = self._port._path_to_user_cache_directory(self._driver_user_directory_suffix)
379         if user_cache_directory:
380             self._target_host.filesystem.maybe_make_directory(user_cache_directory)
381             self._driver_user_cache_directory = user_cache_directory
382         environment = self._setup_environ_for_test()
383         self._crashed_process_name = None
384         self._crashed_pid = None
385         self._server_process = self._port._test_runner_process_constructor(self._port, self._server_name, self.cmd_line(pixel_tests, per_test_args), environment, target_host=self._target_host)
386         self._server_process.start()
387
388     def _run_post_start_tasks(self):
389         # Remote drivers may override this to delay post-start tasks until the server has ack'd.
390         if self._profiler:
391             self._profiler.attach_to_pid(self._pid_on_target())
392
393     def _pid_on_target(self):
394         # Remote drivers will override this method to return the pid on the device.
395         return self._server_process.pid()
396
397     def stop(self):
398         if self._server_process:
399             self._server_process.stop(self._port.driver_stop_timeout())
400             self._server_process = None
401             if self._profiler:
402                 self._profiler.profile_after_exit()
403
404         if self._driver_tempdir:
405             self._target_host.filesystem.rmtree(str(self._driver_tempdir))
406             self._driver_tempdir = None
407         if self._driver_user_cache_directory:
408             self._target_host.filesystem.rmtree(self._driver_user_cache_directory)
409             self._driver_user_cache_directory = None
410
411     def cmd_line(self, pixel_tests, per_test_args):
412         cmd = self._command_wrapper()
413         cmd.append(self._port._path_to_driver())
414         if self._port.get_option('gc_between_tests'):
415             cmd.append('--gc-between-tests')
416         if self._port.get_option('complex_text'):
417             cmd.append('--complex-text')
418         if self._port.get_option('accelerated_drawing'):
419             cmd.append('--accelerated-drawing')
420         if self._port.get_option('remote_layer_tree'):
421             cmd.append('--remote-layer-tree')
422         if self._port.get_option('threaded'):
423             cmd.append('--threaded')
424         if self._no_timeout:
425             cmd.append('--no-timeout')
426         if self._port.get_option('show_touches'):
427             cmd.append('--show-touches')
428
429         for allowed_host in self._port.allowed_hosts():
430             cmd.append('--allowed-host')
431             cmd.append(allowed_host)
432
433         cmd.extend(self._port.get_option('additional_drt_flag', []))
434         cmd.extend(self._port.additional_drt_flag())
435
436         cmd.extend(per_test_args)
437
438         cmd.append('-')
439         return cmd
440
441     def _check_for_driver_timeout(self, out_line):
442         if out_line.startswith("#PID UNRESPONSIVE - "):
443             match = re.match('#PID UNRESPONSIVE - (\S+)', out_line)
444             child_process_name = match.group(1) if match else 'WebProcess'
445             match = re.search('pid (\d+)', out_line)
446             child_process_pid = int(match.group(1)) if match else None
447             err_line = 'Wait on notifyDone timed out, process ' + child_process_name + ' pid = ' + str(child_process_pid)
448             self.error_from_test += err_line
449             _log.debug(err_line)
450             if self._port.get_option("sample_on_timeout"):
451                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
452         if out_line == "FAIL: Timed out waiting for notifyDone to be called\n":
453             self._driver_timed_out = True
454
455     def _check_for_address_sanitizer_violation(self, error_line):
456         if "ERROR: AddressSanitizer" in error_line:
457             return True
458
459     def _check_for_driver_crash_or_unresponsiveness(self, error_line):
460         crashed_check = error_line.rstrip('\r\n')
461         if crashed_check == "#CRASHED":
462             self._crashed_process_name = self._server_process.process_name()
463             self._crashed_pid = self._server_process.pid()
464             return True
465         elif error_line.startswith("#CRASHED - "):
466             match = re.match('#CRASHED - (\S+)', error_line)
467             self._crashed_process_name = match.group(1) if match else 'WebProcess'
468             match = re.search('pid (\d+)', error_line)
469             self._crashed_pid = int(match.group(1)) if match else None
470             _log.debug('%s crash, pid = %s' % (self._crashed_process_name, str(self._crashed_pid)))
471             return True
472         elif error_line.startswith("#PROCESS UNRESPONSIVE - "):
473             match = re.match('#PROCESS UNRESPONSIVE - (\S+)', error_line)
474             child_process_name = match.group(1) if match else 'WebProcess'
475             match = re.search('pid (\d+)', error_line)
476             child_process_pid = int(match.group(1)) if match else None
477             _log.debug('%s is unresponsive, pid = %s' % (child_process_name, str(child_process_pid)))
478             self._driver_timed_out = True
479             if child_process_pid:
480                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
481             self.error_from_test += error_line
482             self._server_process.write('#SAMPLE FINISHED\n', True)  # Must be able to ignore a broken pipe here, target process may already be closed.
483             return True
484         return self.has_crashed()
485
486     def _command_from_driver_input(self, driver_input):
487         # FIXME: performance tests pass in full URLs instead of test names.
488         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
489             command = driver_input.test_name
490         elif self.is_web_platform_test(driver_input.test_name) or self.is_webkit_specific_web_platform_test(driver_input.test_name) or self.is_http_test(driver_input.test_name):
491             command = self.test_to_uri(driver_input.test_name)
492             command += "'--absolutePath'"
493             command += self._port.abspath_for_test(driver_input.test_name, self._target_host)
494         else:
495             command = self._port.abspath_for_test(driver_input.test_name, self._target_host)
496             if sys.platform == 'cygwin':
497                 command = path.cygpath(command)
498
499         assert not driver_input.image_hash or driver_input.should_run_pixel_test
500
501         # ' is the separator between arguments.
502         if self._port.supports_per_test_timeout():
503             command += "'--timeout'%s" % driver_input.timeout
504         if driver_input.should_run_pixel_test:
505             command += "'--pixel-test"
506         if driver_input.should_dump_jsconsolelog_in_stderr:
507             command += "'--dump-jsconsolelog-in-stderr"
508         if driver_input.image_hash:
509             command += "'" + driver_input.image_hash
510         return command + "\n"
511
512     def _read_first_block(self, deadline, test_name):
513         # returns (text_content, audio_content)
514         block = self._read_block(deadline, test_name)
515         if block.malloc:
516             self._measurements['Malloc'] = float(block.malloc)
517         if block.js_heap:
518             self._measurements['JSHeap'] = float(block.js_heap)
519         if block.content_type == 'audio/wav':
520             return (None, block.decoded_content)
521         return (block.decoded_content, None)
522
523     def _read_optional_image_block(self, deadline, test_name):
524         # returns (image, actual_image_hash)
525         block = self._read_block(deadline, test_name, wait_for_stderr_eof=True)
526         if block.content and block.content_type == 'image/png':
527             return (block.decoded_content, block.content_hash)
528         return (None, block.content_hash)
529
530     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
531         if line.startswith(header_text) and getattr(block, header_attr) is None:
532             value = line.split()[1]
533             if header_filter:
534                 value = header_filter(value)
535             setattr(block, header_attr, value)
536             return True
537         return False
538
539     def _process_stdout_line(self, block, line):
540         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
541             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
542             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
543             or self._read_header(block, line, 'ActualHash: ', 'content_hash')
544             or self._read_header(block, line, 'DumpMalloc: ', 'malloc')
545             or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')):
546             return
547         # Note, we're not reading ExpectedHash: here, but we could.
548         # If the line wasn't a header, we just append it to the content.
549         block.content += line
550
551     def _strip_eof(self, line):
552         if line and line.endswith("#EOF\n"):
553             return line[:-5], True
554         return line, False
555
556     def _read_block(self, deadline, test_name, wait_for_stderr_eof=False):
557         block = ContentBlock()
558         out_seen_eof = False
559         asan_violation_detected = False
560
561         while True:
562             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
563                 break
564
565             if self.err_seen_eof:
566                 out_line = self._server_process.read_stdout_line(deadline)
567                 err_line = None
568             elif out_seen_eof:
569                 out_line = None
570                 err_line = self._server_process.read_stderr_line(deadline)
571             else:
572                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
573
574             # ServerProcess returns None for time outs and crashes.
575             if out_line is None and err_line is None:
576                 break
577
578             if out_line:
579                 assert not out_seen_eof
580                 out_line, out_seen_eof = self._strip_eof(out_line)
581             if err_line:
582                 assert not self.err_seen_eof
583                 err_line, self.err_seen_eof = self._strip_eof(err_line)
584
585             if out_line:
586                 self._check_for_driver_timeout(out_line)
587                 if out_line[-1] != "\n":
588                     _log.error("  %s -> Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug." % test_name)
589                 content_length_before_header_check = block._content_length
590                 self._process_stdout_line(block, out_line)
591                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
592                 # Don't wait until we're done with headers, just read the binary blob right now.
593                 if content_length_before_header_check != block._content_length:
594                     block.content = self._server_process.read_stdout(deadline, block._content_length)
595
596             if err_line:
597                 if self._check_for_driver_crash_or_unresponsiveness(err_line):
598                     break
599                 elif self._check_for_address_sanitizer_violation(err_line):
600                     asan_violation_detected = True
601                     self._crash_report_from_driver = ""
602                     # ASan report starts with a nondescript line, we only detect the second line.
603                     end_of_previous_error_line = self.error_from_test.rfind('\n', 0, -1)
604                     if end_of_previous_error_line > 0:
605                         self.error_from_test = self.error_from_test[:end_of_previous_error_line]
606                     else:
607                         self.error_from_test = ""
608                     # Symbolication can take a very long time, give it 10 extra minutes to finish.
609                     # FIXME: This can likely be removed once <rdar://problem/18701447> is fixed.
610                     deadline += 10 * 60 * 1000
611                 if asan_violation_detected:
612                     self._crash_report_from_driver += err_line
613                 else:
614                     self.error_from_test += err_line
615
616         if asan_violation_detected and not self._crashed_process_name:
617             self._crashed_process_name = self._server_process.process_name()
618             self._crashed_pid = self._server_process.pid()
619
620         block.decode_content()
621         return block
622
623     @staticmethod
624     def check_driver(port):
625         # This checks if the required system dependencies for the driver are met.
626         # Since this is the generic class implementation, just return True.
627         return True
628
629
630 class ContentBlock(object):
631     def __init__(self):
632         self.content_type = None
633         self.encoding = None
634         self.content_hash = None
635         self._content_length = None
636         # Content is treated as binary data even though the text output is usually UTF-8.
637         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
638         self.decoded_content = None
639         self.malloc = None
640         self.js_heap = None
641
642     def decode_content(self):
643         if self.encoding == 'base64' and self.content is not None:
644             self.decoded_content = base64.b64decode(self.content)
645         else:
646             self.decoded_content = self.content
647
648
649 class DriverProxy(object):
650     """A wrapper for managing two Driver instances, one with pixel tests and
651     one without. This allows us to handle plain text tests and ref tests with a
652     single driver."""
653
654     def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
655         self._port = port
656         self._worker_number = worker_number
657         self._driver_instance_constructor = driver_instance_constructor
658         self._no_timeout = no_timeout
659
660         # FIXME: We shouldn't need to create a driver until we actually run a test.
661         self._driver = self._make_driver(pixel_tests)
662         self._driver_cmd_line = None
663
664     def _make_driver(self, pixel_tests):
665         return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)
666
667     # FIXME: this should be a @classmethod (or implemented on Port instead).
668     def is_http_test(self, test_name):
669         return self._driver.is_http_test(test_name)
670
671     def is_web_platform_test(self, test_name):
672         return self._driver.is_web_platform_test(test_name)
673
674     def is_webkit_specific_web_platform_test(self, test_name):
675         return self._driver.is_webkit_specific_web_platform_test(test_name)
676
677     # FIXME: this should be a @classmethod (or implemented on Port instead).
678     def test_to_uri(self, test_name):
679         return self._driver.test_to_uri(test_name)
680
681     # FIXME: this should be a @classmethod (or implemented on Port instead).
682     def uri_to_test(self, uri):
683         return self._driver.uri_to_test(uri)
684
685     def run_test(self, driver_input, stop_when_done):
686         pixel_tests_needed = driver_input.should_run_pixel_test
687         cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
688         if cmd_line_key != self._driver_cmd_line:
689             self._driver.stop()
690             self._driver = self._make_driver(pixel_tests_needed)
691             self._driver_cmd_line = cmd_line_key
692
693         return self._driver.run_test(driver_input, stop_when_done)
694
695     def has_crashed(self):
696         return self._driver.has_crashed()
697
698     def stop(self):
699         self._driver.stop()
700
701     # FIXME: this should be a @classmethod (or implemented on Port instead).
702     def cmd_line(self, pixel_tests=None, per_test_args=None):
703         return self._driver.cmd_line(pixel_tests, per_test_args or [])
704
705     def _cmd_line_as_key(self, pixel_tests, per_test_args):
706         return ' '.join(self.cmd_line(pixel_tests, per_test_args))