[WinCairo] Pass correct url for http test to test driver.
[WebKit-https.git] / Tools / Scripts / webkitpy / port / driver.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2015, 2016 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import base64
31 import logging
32 import re
33 import shlex
34 import sys
35 import time
36 import os
37
38 from os.path import normpath
39 from webkitpy.common.system import path
40 from webkitpy.common.system.profiler import ProfilerFactory
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class DriverInput(object):
47     def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, should_dump_jsconsolelog_in_stderr=None, args=None):
48         self.test_name = test_name
49         self.timeout = timeout  # in ms
50         self.image_hash = image_hash
51         self.should_run_pixel_test = should_run_pixel_test
52         self.should_dump_jsconsolelog_in_stderr = should_dump_jsconsolelog_in_stderr
53         self.args = args or []
54
55     def __repr__(self):
56         return "DriverInput(test_name='{}', timeout={}, image_hash={}, should_run_pixel_test={}, should_dump_jsconsolelog_in_stderr={}'".format(self.test_name, self.timeout, self.image_hash, self.should_run_pixel_test, self.should_dump_jsconsolelog_in_stderr)
57
58
59 class DriverOutput(object):
60     """Groups information about a output from driver for easy passing
61     and post-processing of data."""
62
63     metrics_patterns = []
64     metrics_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
65     metrics_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
66     metrics_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
67     metrics_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
68     metrics_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
69     metrics_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
70     metrics_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
71     metrics_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
72     metrics_patterns.append((re.compile('\([0-9]+px'), 'px'))
73     metrics_patterns.append((re.compile(' *" *\n +" *'), ' '))
74     metrics_patterns.append((re.compile('" +$'), '"'))
75     metrics_patterns.append((re.compile('- '), '-'))
76     metrics_patterns.append((re.compile('\n( *)"\s+'), '\n\g<1>"'))
77     metrics_patterns.append((re.compile('\s+"\n'), '"\n'))
78     metrics_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
79     metrics_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))
80     metrics_patterns.append((re.compile('scrollX [0-9]+'), 'scrollX'))
81     metrics_patterns.append((re.compile('scrollY [0-9]+'), 'scrollY'))
82     metrics_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled'))
83
84     def __init__(self, text, image, image_hash, audio, crash=False,
85             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
86             crashed_pid=None, crash_log=None, pid=None):
87         # FIXME: Args could be renamed to better clarify what they do.
88         self.text = text
89         self.image = image  # May be empty-string if the test crashes.
90         self.image_hash = image_hash
91         self.image_diff = None  # image_diff gets filled in after construction.
92         self.audio = audio  # Binary format is port-dependent.
93         self.crash = crash
94         self.crashed_process_name = crashed_process_name
95         self.crashed_pid = crashed_pid
96         self.crash_log = crash_log
97         self.test_time = test_time
98         self.measurements = measurements
99         self.timeout = timeout
100         self.error = error  # stderr output
101         self.pid = pid
102
103     def has_stderr(self):
104         return bool(self.error)
105
106     def strip_metrics(self):
107         self.strip_patterns(self.metrics_patterns)
108
109     def strip_patterns(self, patterns):
110         if not self.text:
111             return
112         for pattern in patterns:
113             self.text = re.sub(pattern[0], pattern[1], self.text)
114
115     def strip_stderror_patterns(self, patterns):
116         if not self.error:
117             return
118         for pattern in patterns:
119             self.error = re.sub(pattern[0], pattern[1], self.error)
120
121
122 class Driver(object):
123     """object for running test(s) using DumpRenderTree/WebKitTestRunner."""
124
125     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
126         """Initialize a Driver to subsequently run tests.
127
128         Typically this routine will spawn DumpRenderTree in a config
129         ready for subsequent input.
130
131         port - reference back to the port object.
132         worker_number - identifier for a particular worker/driver instance
133         """
134         self._port = port
135         self._worker_number = worker_number
136         self._no_timeout = no_timeout
137         self._target_host = port.target_host(worker_number)
138
139         self._driver_tempdir = None
140         self._driver_user_directory_suffix = None
141         self._driver_user_cache_directory = None
142
143         # WebKitTestRunner can report back subprocess crashes by printing
144         # "#CRASHED - PROCESSNAME".  Since those can happen at any time and ServerProcess
145         # won't be aware of them (since the actual tool didn't crash, just a subprocess)
146         # we record the crashed subprocess name here.
147         self._crashed_process_name = None
148         self._crashed_pid = None
149
150         self._driver_timed_out = False
151
152         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
153         # stderr output, as well as if we've seen #EOF on this driver instance.
154         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
155         # instead scope these locally in run_test.
156         self.error_from_test = str()
157         self.err_seen_eof = False
158
159         self._server_name = self._port.driver_name()
160         self._server_process = None
161
162         self._measurements = {}
163         if self._port.get_option("profile"):
164             profiler_name = self._port.get_option("profiler")
165             self._profiler = ProfilerFactory.create_profiler(self._port.host,
166                 self._port._path_to_driver(), self._port.results_directory(), profiler_name)
167         else:
168             self._profiler = None
169
170         self.web_platform_test_server_doc_root = self._port.web_platform_test_server_doc_root()
171         self.web_platform_test_server_base_http_url = self._port.web_platform_test_server_base_http_url()
172         self.web_platform_test_server_base_https_url = self._port.web_platform_test_server_base_https_url()
173
174     def __del__(self):
175         self.stop()
176
177     def run_test(self, driver_input, stop_when_done):
178         """Run a single test and return the results.
179
180         Note that it is okay if a test times out or crashes and leaves
181         the driver in an indeterminate state. The upper layers of the program
182         are responsible for cleaning up and ensuring things are okay.
183
184         Returns a DriverOutput object.
185         """
186         start_time = time.time()
187         self.start(driver_input.should_run_pixel_test, driver_input.args)
188         test_begin_time = time.time()
189         self._driver_timed_out = False
190         self._crash_report_from_driver = None
191         self.error_from_test = str()
192         self.err_seen_eof = False
193
194         command = self._command_from_driver_input(driver_input)
195
196         # Certain timeouts are detected by the tool itself; tool detection is better,
197         # because results contain partial output in this case. Make script timeout longer
198         # by 5 seconds to avoid racing for which timeout is detected first.
199         # FIXME: It's not the job of the driver to decide what the timeouts should be.
200         # Move the additional timeout to driver_input.
201         if self._no_timeout:
202             deadline = test_begin_time + 60 * 60 * 24 * 7  # 7 days. Using sys.maxint causes a hang.
203         else:
204             deadline = test_begin_time + int(driver_input.timeout) / 1000.0 + 5
205
206         self._server_process.write(command)
207         text, audio = self._read_first_block(deadline, driver_input.test_name)  # First block is either text or audio
208         image, actual_image_hash = self._read_optional_image_block(deadline, driver_input.test_name)  # The second (optional) block is image data.
209
210         crashed = self.has_crashed()
211         timed_out = self._server_process.timed_out
212         driver_timed_out = self._driver_timed_out
213         pid = self._server_process.pid()
214
215         if stop_when_done or crashed or timed_out:
216             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
217             # In the timeout case, we kill the hung process as well.
218             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
219             if out:
220                 text += out
221             if err:
222                 self.error_from_test += err
223             self._server_process = None
224
225         crash_log = None
226         if self._crash_report_from_driver:
227             crash_log = self._crash_report_from_driver
228         elif crashed:
229             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
230             # If we don't find a crash log use a placeholder error message instead.
231             if not crash_log:
232                 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid"
233                 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str)
234
235                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
236                 if self.error_from_test:
237                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
238
239         return DriverOutput(text, image, actual_image_hash, audio,
240             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
241             timeout=timed_out or driver_timed_out, error=self.error_from_test,
242             crashed_process_name=self._crashed_process_name,
243             crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
244
245     def _get_crash_log(self, stdout, stderr, newer_than):
246         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than, target_host=self._target_host)
247
248     def _command_wrapper(self):
249         # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
250         wrapper_arguments = []
251         if self._profiler:
252             wrapper_arguments = self._profiler.wrapper_arguments()
253         if self._port.get_option('wrapper'):
254             return shlex.split(self._port.get_option('wrapper')) + wrapper_arguments
255         return wrapper_arguments
256
257     HTTP_DIR = normpath("http/tests") + os.sep
258     HTTP_LOCAL_DIR = normpath("http/tests/local") + os.sep
259     WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR = normpath("http/wpt") + os.sep
260     WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE = normpath("WebKit") + os.sep
261
262     def is_http_test(self, test_name):
263         return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)
264
265     def is_webkit_specific_web_platform_test(self, test_name):
266         return test_name.startswith(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
267
268     def is_web_platform_test(self, test_name):
269         return test_name.startswith(self.web_platform_test_server_doc_root)
270
271     def wpt_test_path_to_uri(self, path):
272         return self.web_platform_test_server_base_https_url + path if ".https." in path else self.web_platform_test_server_base_http_url + path
273
274     def http_test_path_to_uri(self, path):
275         path = path.replace(os.sep, '/')
276         return self.http_base_url(secure=self.is_secure_path(path)) + path
277
278     def is_secure_path(self, path):
279         return path.startswith("ssl") or ".https." in path
280
281     def http_base_url(self, secure=None):
282         return "%s://127.0.0.1:%d/" % (('https', 8443) if secure else ('http', 8000))
283
284     def test_to_uri(self, test_name):
285         """Convert a test name to a URI."""
286         if self.is_web_platform_test(test_name):
287             return self.wpt_test_path_to_uri(test_name[len(self.web_platform_test_server_doc_root):])
288         if self.is_webkit_specific_web_platform_test(test_name):
289             return self.wpt_test_path_to_uri(self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE + test_name[len(self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR):])
290
291         if not self.is_http_test(test_name):
292             return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))
293
294         return self.http_test_path_to_uri(test_name[len(self.HTTP_DIR):])
295
296     def uri_to_test(self, uri):
297         """Return the base layout test name for a given URI.
298
299         This returns the test name for a given URI, e.g., if you passed in
300         "file:///src/LayoutTests/fast/html/keygen.html" it would return
301         "fast/html/keygen.html".
302
303         """
304         if uri.startswith("file:///"):
305             prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
306             if not prefix.endswith('/'):
307                 prefix += '/'
308             return uri[len(prefix):]
309         if uri.startswith(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
310             return uri.replace(self.web_platform_test_server_base_http_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
311         if uri.startswith(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE):
312             return uri.replace(self.web_platform_test_server_base_https_url + self.WEBKIT_WEB_PLATFORM_TEST_SERVER_ROUTE, self.WEBKIT_SPECIFIC_WEB_PLATFORM_TEST_SUBDIR)
313         if uri.startswith(self.web_platform_test_server_base_http_url):
314             return uri.replace(self.web_platform_test_server_base_http_url, self.web_platform_test_server_doc_root)
315         if uri.startswith(self.web_platform_test_server_base_https_url):
316             return uri.replace(self.web_platform_test_server_base_https_url, self.web_platform_test_server_doc_root)
317         if uri.startswith("http://"):
318             return uri.replace(self.http_base_url(secure=False), self.HTTP_DIR)
319         if uri.startswith("https://"):
320             return uri.replace(self.http_base_url(secure=True), self.HTTP_DIR)
321         raise NotImplementedError('unknown url type: %s' % uri)
322
323     def has_crashed(self):
324         if self._server_process is None:
325             return False
326         if self._crashed_process_name:
327             return True
328         if self._server_process.has_crashed():
329             self._crashed_process_name = self._server_process.process_name()
330             self._crashed_pid = self._server_process.pid()
331             return True
332         return False
333
334     def start(self, pixel_tests, per_test_args):
335         # FIXME: Callers shouldn't normally call this, since this routine
336         # may not be specifying the correct combination of pixel test and
337         # per_test args.
338         #
339         # The only reason we have this routine at all is so the perftestrunner
340         # can pause before running a test; it might be better to push that
341         # into run_test() directly.
342         if not self._server_process:
343             self._start(pixel_tests, per_test_args)
344             self._run_post_start_tasks()
345
346     def _append_environment_variable_path(self, environment, variable, path):
347         if variable in environment:
348             environment[variable] = environment[variable] + os.pathsep + path
349         else:
350             environment[variable] = path
351
352     def _setup_environ_for_driver(self, environment):
353         build_root_path = str(self._port._build_path())
354         self._append_environment_variable_path(environment, 'DYLD_LIBRARY_PATH', build_root_path)
355         self._append_environment_variable_path(environment, '__XPC_DYLD_LIBRARY_PATH', build_root_path)
356         self._append_environment_variable_path(environment, 'DYLD_FRAMEWORK_PATH', build_root_path)
357         self._append_environment_variable_path(environment, '__XPC_DYLD_FRAMEWORK_PATH', build_root_path)
358         # Use an isolated temp directory that can be deleted after testing (especially important on Mac, as
359         # CoreMedia disk cache is in the temp directory).
360         environment['TMPDIR'] = str(self._driver_tempdir)
361         environment['DIRHELPER_USER_DIR_SUFFIX'] = self._driver_user_directory_suffix
362         # Put certain normally persistent files into the temp directory (e.g. IndexedDB storage).
363         if sys.platform == 'cygwin':
364             environment['DUMPRENDERTREE_TEMP'] = path.cygpath(str(self._driver_tempdir))
365         else:
366             environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
367         environment['LOCAL_RESOURCE_ROOT'] = str(self._port.layout_tests_dir())
368         environment['ASAN_OPTIONS'] = "allocator_may_return_null=1"
369         environment['__XPC_ASAN_OPTIONS'] = environment['ASAN_OPTIONS']
370         if self._profiler:
371             environment = self._profiler.adjusted_environment(environment)
372         return environment
373
374     def _setup_environ_for_test(self):
375         environment = self._port.setup_environ_for_server(self._server_name)
376         environment = self._setup_environ_for_driver(environment)
377         return environment
378
379     def _start(self, pixel_tests, per_test_args):
380         self.stop()
381         # Each driver process should be using individual directories under _driver_tempdir (which is deleted when stopping),
382         # however some subsystems on some platforms could end up using process default ones.
383         self._port._clear_global_caches_and_temporary_files()
384         self._driver_tempdir = self._port._driver_tempdir(self._target_host)
385         self._driver_user_directory_suffix = os.path.basename(str(self._driver_tempdir))
386         user_cache_directory = self._port._path_to_user_cache_directory(self._driver_user_directory_suffix)
387         if user_cache_directory:
388             self._target_host.filesystem.maybe_make_directory(user_cache_directory)
389             self._driver_user_cache_directory = user_cache_directory
390         environment = self._setup_environ_for_test()
391         self._crashed_process_name = None
392         self._crashed_pid = None
393         self._server_process = self._port._test_runner_process_constructor(self._port, self._server_name, self.cmd_line(pixel_tests, per_test_args), environment, target_host=self._target_host)
394         self._server_process.start()
395
396     def _run_post_start_tasks(self):
397         # Remote drivers may override this to delay post-start tasks until the server has ack'd.
398         if self._profiler:
399             self._profiler.attach_to_pid(self._pid_on_target())
400
401     def _pid_on_target(self):
402         # Remote drivers will override this method to return the pid on the device.
403         return self._server_process.pid()
404
405     def stop(self):
406         if self._server_process:
407             self._server_process.stop(self._port.driver_stop_timeout())
408             self._server_process = None
409             if self._profiler:
410                 self._profiler.profile_after_exit()
411
412         if self._driver_tempdir:
413             self._target_host.filesystem.rmtree(str(self._driver_tempdir))
414             self._driver_tempdir = None
415         if self._driver_user_cache_directory:
416             self._target_host.filesystem.rmtree(self._driver_user_cache_directory)
417             self._driver_user_cache_directory = None
418
419     def cmd_line(self, pixel_tests, per_test_args):
420         cmd = self._command_wrapper()
421         cmd.append(self._port._path_to_driver())
422         if self._port.get_option('gc_between_tests'):
423             cmd.append('--gc-between-tests')
424         if self._port.get_option('complex_text'):
425             cmd.append('--complex-text')
426         if self._port.get_option('accelerated_drawing'):
427             cmd.append('--accelerated-drawing')
428         if self._port.get_option('remote_layer_tree'):
429             cmd.append('--remote-layer-tree')
430         if self._port.get_option('threaded'):
431             cmd.append('--threaded')
432         if self._no_timeout:
433             cmd.append('--no-timeout')
434         if self._port.get_option('show_touches'):
435             cmd.append('--show-touches')
436
437         for allowed_host in self._port.allowed_hosts():
438             cmd.append('--allowed-host')
439             cmd.append(allowed_host)
440
441         cmd.extend(self._port.get_option('additional_drt_flag', []))
442         cmd.extend(self._port.additional_drt_flag())
443
444         cmd.extend(per_test_args)
445
446         cmd.append('-')
447         return cmd
448
449     def _check_for_driver_timeout(self, out_line):
450         if out_line.startswith("#PID UNRESPONSIVE - "):
451             match = re.match('#PID UNRESPONSIVE - (\S+)', out_line)
452             child_process_name = match.group(1) if match else 'WebProcess'
453             match = re.search('pid (\d+)', out_line)
454             child_process_pid = int(match.group(1)) if match else None
455             err_line = 'Wait on notifyDone timed out, process ' + child_process_name + ' pid = ' + str(child_process_pid)
456             self.error_from_test += err_line
457             _log.debug(err_line)
458             if self._port.get_option("sample_on_timeout"):
459                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
460         if out_line == "FAIL: Timed out waiting for notifyDone to be called\n":
461             self._driver_timed_out = True
462
463     def _check_for_address_sanitizer_violation(self, error_line):
464         if "ERROR: AddressSanitizer" in error_line:
465             return True
466
467     def _check_for_driver_crash_or_unresponsiveness(self, error_line):
468         crashed_check = error_line.rstrip('\r\n')
469         if crashed_check == "#CRASHED":
470             self._crashed_process_name = self._server_process.process_name()
471             self._crashed_pid = self._server_process.pid()
472             return True
473         elif error_line.startswith("#CRASHED - "):
474             match = re.match('#CRASHED - (\S+)', error_line)
475             self._crashed_process_name = match.group(1) if match else 'WebProcess'
476             match = re.search('pid (\d+)', error_line)
477             self._crashed_pid = int(match.group(1)) if match else None
478             _log.debug('%s crash, pid = %s' % (self._crashed_process_name, str(self._crashed_pid)))
479             return True
480         elif error_line.startswith("#PROCESS UNRESPONSIVE - "):
481             match = re.match('#PROCESS UNRESPONSIVE - (\S+)', error_line)
482             child_process_name = match.group(1) if match else 'WebProcess'
483             match = re.search('pid (\d+)', error_line)
484             child_process_pid = int(match.group(1)) if match else None
485             _log.debug('%s is unresponsive, pid = %s' % (child_process_name, str(child_process_pid)))
486             self._driver_timed_out = True
487             if child_process_pid:
488                 self._port.sample_process(child_process_name, child_process_pid, self._target_host)
489             self.error_from_test += error_line
490             self._server_process.write('#SAMPLE FINISHED\n', True)  # Must be able to ignore a broken pipe here, target process may already be closed.
491             return True
492         return self.has_crashed()
493
494     def _command_from_driver_input(self, driver_input):
495         # FIXME: performance tests pass in full URLs instead of test names.
496         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
497             command = driver_input.test_name
498         elif self.is_web_platform_test(driver_input.test_name) or self.is_webkit_specific_web_platform_test(driver_input.test_name) or self.is_http_test(driver_input.test_name):
499             command = self.test_to_uri(driver_input.test_name)
500             command += "'--absolutePath'"
501             command += self._port.abspath_for_test(driver_input.test_name, self._target_host)
502         else:
503             command = self._port.abspath_for_test(driver_input.test_name, self._target_host)
504             if sys.platform == 'cygwin':
505                 command = path.cygpath(command)
506
507         assert not driver_input.image_hash or driver_input.should_run_pixel_test
508
509         # ' is the separator between arguments.
510         if self._port.supports_per_test_timeout():
511             command += "'--timeout'%s" % driver_input.timeout
512         if driver_input.should_run_pixel_test:
513             command += "'--pixel-test"
514         if driver_input.should_dump_jsconsolelog_in_stderr:
515             command += "'--dump-jsconsolelog-in-stderr"
516         if driver_input.image_hash:
517             command += "'" + driver_input.image_hash
518         return command + "\n"
519
520     def _read_first_block(self, deadline, test_name):
521         # returns (text_content, audio_content)
522         block = self._read_block(deadline, test_name)
523         if block.malloc:
524             self._measurements['Malloc'] = float(block.malloc)
525         if block.js_heap:
526             self._measurements['JSHeap'] = float(block.js_heap)
527         if block.content_type == 'audio/wav':
528             return (None, block.decoded_content)
529         return (block.decoded_content, None)
530
531     def _read_optional_image_block(self, deadline, test_name):
532         # returns (image, actual_image_hash)
533         block = self._read_block(deadline, test_name, wait_for_stderr_eof=True)
534         if block.content and block.content_type == 'image/png':
535             return (block.decoded_content, block.content_hash)
536         return (None, block.content_hash)
537
538     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
539         if line.startswith(header_text) and getattr(block, header_attr) is None:
540             value = line.split()[1]
541             if header_filter:
542                 value = header_filter(value)
543             setattr(block, header_attr, value)
544             return True
545         return False
546
547     def _process_stdout_line(self, block, line):
548         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
549             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
550             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
551             or self._read_header(block, line, 'ActualHash: ', 'content_hash')
552             or self._read_header(block, line, 'DumpMalloc: ', 'malloc')
553             or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')):
554             return
555         # Note, we're not reading ExpectedHash: here, but we could.
556         # If the line wasn't a header, we just append it to the content.
557         block.content += line
558
559     def _strip_eof(self, line):
560         if line and line.endswith("#EOF\n"):
561             return line[:-5], True
562         return line, False
563
564     def _read_block(self, deadline, test_name, wait_for_stderr_eof=False):
565         block = ContentBlock()
566         out_seen_eof = False
567         asan_violation_detected = False
568
569         while True:
570             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
571                 break
572
573             if self.err_seen_eof:
574                 out_line = self._server_process.read_stdout_line(deadline)
575                 err_line = None
576             elif out_seen_eof:
577                 out_line = None
578                 err_line = self._server_process.read_stderr_line(deadline)
579             else:
580                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
581
582             # ServerProcess returns None for time outs and crashes.
583             if out_line is None and err_line is None:
584                 break
585
586             if out_line:
587                 assert not out_seen_eof
588                 out_line, out_seen_eof = self._strip_eof(out_line)
589             if err_line:
590                 assert not self.err_seen_eof
591                 err_line, self.err_seen_eof = self._strip_eof(err_line)
592
593             if out_line:
594                 self._check_for_driver_timeout(out_line)
595                 if out_line[-1] != "\n":
596                     _log.error("  %s -> Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug." % test_name)
597                 content_length_before_header_check = block._content_length
598                 self._process_stdout_line(block, out_line)
599                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
600                 # Don't wait until we're done with headers, just read the binary blob right now.
601                 if content_length_before_header_check != block._content_length:
602                     block.content = self._server_process.read_stdout(deadline, block._content_length)
603
604             if err_line:
605                 if self._check_for_driver_crash_or_unresponsiveness(err_line):
606                     break
607                 elif self._check_for_address_sanitizer_violation(err_line):
608                     asan_violation_detected = True
609                     self._crash_report_from_driver = ""
610                     # ASan report starts with a nondescript line, we only detect the second line.
611                     end_of_previous_error_line = self.error_from_test.rfind('\n', 0, -1)
612                     if end_of_previous_error_line > 0:
613                         self.error_from_test = self.error_from_test[:end_of_previous_error_line]
614                     else:
615                         self.error_from_test = ""
616                     # Symbolication can take a very long time, give it 10 extra minutes to finish.
617                     # FIXME: This can likely be removed once <rdar://problem/18701447> is fixed.
618                     deadline += 10 * 60 * 1000
619                 if asan_violation_detected:
620                     self._crash_report_from_driver += err_line
621                 else:
622                     self.error_from_test += err_line
623
624         if asan_violation_detected and not self._crashed_process_name:
625             self._crashed_process_name = self._server_process.process_name()
626             self._crashed_pid = self._server_process.pid()
627
628         block.decode_content()
629         return block
630
631     @staticmethod
632     def check_driver(port):
633         # This checks if the required system dependencies for the driver are met.
634         # Since this is the generic class implementation, just return True.
635         return True
636
637
638 class ContentBlock(object):
639     def __init__(self):
640         self.content_type = None
641         self.encoding = None
642         self.content_hash = None
643         self._content_length = None
644         # Content is treated as binary data even though the text output is usually UTF-8.
645         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
646         self.decoded_content = None
647         self.malloc = None
648         self.js_heap = None
649
650     def decode_content(self):
651         if self.encoding == 'base64' and self.content is not None:
652             self.decoded_content = base64.b64decode(self.content)
653         else:
654             self.decoded_content = self.content
655
656
657 class DriverProxy(object):
658     """A wrapper for managing two Driver instances, one with pixel tests and
659     one without. This allows us to handle plain text tests and ref tests with a
660     single driver."""
661
662     def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
663         self._port = port
664         self._worker_number = worker_number
665         self._driver_instance_constructor = driver_instance_constructor
666         self._no_timeout = no_timeout
667
668         # FIXME: We shouldn't need to create a driver until we actually run a test.
669         self._driver = self._make_driver(pixel_tests)
670         self._driver_cmd_line = None
671
672     def _make_driver(self, pixel_tests):
673         return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)
674
675     # FIXME: this should be a @classmethod (or implemented on Port instead).
676     def is_http_test(self, test_name):
677         return self._driver.is_http_test(test_name)
678
679     def is_web_platform_test(self, test_name):
680         return self._driver.is_web_platform_test(test_name)
681
682     def is_webkit_specific_web_platform_test(self, test_name):
683         return self._driver.is_webkit_specific_web_platform_test(test_name)
684
685     # FIXME: this should be a @classmethod (or implemented on Port instead).
686     def test_to_uri(self, test_name):
687         return self._driver.test_to_uri(test_name)
688
689     # FIXME: this should be a @classmethod (or implemented on Port instead).
690     def uri_to_test(self, uri):
691         return self._driver.uri_to_test(uri)
692
693     def run_test(self, driver_input, stop_when_done):
694         pixel_tests_needed = driver_input.should_run_pixel_test
695         cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
696         if cmd_line_key != self._driver_cmd_line:
697             self._driver.stop()
698             self._driver = self._make_driver(pixel_tests_needed)
699             self._driver_cmd_line = cmd_line_key
700
701         return self._driver.run_test(driver_input, stop_when_done)
702
703     def has_crashed(self):
704         return self._driver.has_crashed()
705
706     def stop(self):
707         self._driver.stop()
708
709     # FIXME: this should be a @classmethod (or implemented on Port instead).
710     def cmd_line(self, pixel_tests=None, per_test_args=None):
711         return self._driver.cmd_line(pixel_tests, per_test_args or [])
712
713     def _cmd_line_as_key(self, pixel_tests, per_test_args):
714         return ' '.join(self.cmd_line(pixel_tests, per_test_args))