8422433eb2fa397219ffec72543cddf2eaca92db
[WebKit-https.git] / Tools / Scripts / webkitpy / port / driver.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import base64
30 import copy
31 import logging
32 import re
33 import shlex
34 import sys
35 import time
36 import os
37
38 from webkitpy.common.system import path
39 from webkitpy.common.system.profiler import ProfilerFactory
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class DriverInput(object):
46     def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, args=None):
47         self.test_name = test_name
48         self.timeout = timeout  # in ms
49         self.image_hash = image_hash
50         self.should_run_pixel_test = should_run_pixel_test
51         self.args = args or []
52
53
54 class DriverOutput(object):
55     """Groups information about a output from driver for easy passing
56     and post-processing of data."""
57
58     metrics_patterns = []
59     metrics_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
60     metrics_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
61     metrics_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
62     metrics_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
63     metrics_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
64     metrics_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
65     metrics_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
66     metrics_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
67     metrics_patterns.append((re.compile('\([0-9]+px'), 'px'))
68     metrics_patterns.append((re.compile(' *" *\n +" *'), ' '))
69     metrics_patterns.append((re.compile('" +$'), '"'))
70     metrics_patterns.append((re.compile('- '), '-'))
71     metrics_patterns.append((re.compile('\n( *)"\s+'), '\n\g<1>"'))
72     metrics_patterns.append((re.compile('\s+"\n'), '"\n'))
73     metrics_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
74     metrics_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))
75     metrics_patterns.append((re.compile('scrollX [0-9]+'), 'scrollX'))
76     metrics_patterns.append((re.compile('scrollY [0-9]+'), 'scrollY'))
77     metrics_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled'))
78
79     def __init__(self, text, image, image_hash, audio, crash=False,
80             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
81             crashed_pid=None, crash_log=None, pid=None):
82         # FIXME: Args could be renamed to better clarify what they do.
83         self.text = text
84         self.image = image  # May be empty-string if the test crashes.
85         self.image_hash = image_hash
86         self.image_diff = None  # image_diff gets filled in after construction.
87         self.audio = audio  # Binary format is port-dependent.
88         self.crash = crash
89         self.crashed_process_name = crashed_process_name
90         self.crashed_pid = crashed_pid
91         self.crash_log = crash_log
92         self.test_time = test_time
93         self.measurements = measurements
94         self.timeout = timeout
95         self.error = error  # stderr output
96         self.pid = pid
97
98     def has_stderr(self):
99         return bool(self.error)
100
101     def strip_metrics(self):
102         self.strip_patterns(self.metrics_patterns)
103
104     def strip_patterns(self, patterns):
105         if not self.text:
106             return
107         for pattern in patterns:
108             self.text = re.sub(pattern[0], pattern[1], self.text)
109
110     def strip_stderror_patterns(self, patterns):
111         if not self.error:
112             return
113         for pattern in patterns:
114             self.error = re.sub(pattern[0], pattern[1], self.error)
115
116 class Driver(object):
117     """object for running test(s) using DumpRenderTree/WebKitTestRunner."""
118
119     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
120         """Initialize a Driver to subsequently run tests.
121
122         Typically this routine will spawn DumpRenderTree in a config
123         ready for subsequent input.
124
125         port - reference back to the port object.
126         worker_number - identifier for a particular worker/driver instance
127         """
128         self._port = port
129         self._worker_number = worker_number
130         self._no_timeout = no_timeout
131
132         self._driver_tempdir = None
133         # WebKitTestRunner/LayoutTestRelay can report back subprocess crashes by printing
134         # "#CRASHED - PROCESSNAME".  Since those can happen at any time and ServerProcess
135         # won't be aware of them (since the actual tool didn't crash, just a subprocess)
136         # we record the crashed subprocess name here.
137         self._crashed_process_name = None
138         self._crashed_pid = None
139
140         # WebKitTestRunner can report back subprocesses that became unresponsive
141         # This could mean they crashed.
142         self._subprocess_was_unresponsive = False
143
144         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
145         # stderr output, as well as if we've seen #EOF on this driver instance.
146         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
147         # instead scope these locally in run_test.
148         self.error_from_test = str()
149         self.err_seen_eof = False
150         self._server_process = None
151
152         self._measurements = {}
153         if self._port.get_option("profile"):
154             profiler_name = self._port.get_option("profiler")
155             self._profiler = ProfilerFactory.create_profiler(self._port.host,
156                 self._port._path_to_driver(), self._port.results_directory(), profiler_name)
157         else:
158             self._profiler = None
159
160     def __del__(self):
161         self.stop()
162
163     def run_test(self, driver_input, stop_when_done):
164         """Run a single test and return the results.
165
166         Note that it is okay if a test times out or crashes and leaves
167         the driver in an indeterminate state. The upper layers of the program
168         are responsible for cleaning up and ensuring things are okay.
169
170         Returns a DriverOutput object.
171         """
172         start_time = time.time()
173         self.start(driver_input.should_run_pixel_test, driver_input.args)
174         test_begin_time = time.time()
175         self.error_from_test = str()
176         self.err_seen_eof = False
177
178         command = self._command_from_driver_input(driver_input)
179         deadline = test_begin_time + int(driver_input.timeout) / 1000.0
180
181         self._server_process.write(command)
182         text, audio = self._read_first_block(deadline)  # First block is either text or audio
183         image, actual_image_hash = self._read_optional_image_block(deadline)  # The second (optional) block is image data.
184
185         crashed = self.has_crashed()
186         timed_out = self._server_process.timed_out
187         pid = self._server_process.pid()
188
189         if stop_when_done or crashed or timed_out:
190             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
191             # In the timeout case, we kill the hung process as well.
192             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
193             if out:
194                 text += out
195             if err:
196                 self.error_from_test += err
197             self._server_process = None
198
199         crash_log = None
200         if crashed:
201             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
202             # If we don't find a crash log use a placeholder error message instead.
203             if not crash_log:
204                 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid"
205                 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str)
206                 # If we were unresponsive append a message informing there may not have been a crash.
207                 if self._subprocess_was_unresponsive:
208                     crash_log += 'Process failed to become responsive before timing out.\n'
209
210                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
211                 if self.error_from_test:
212                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
213
214         return DriverOutput(text, image, actual_image_hash, audio,
215             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
216             timeout=timed_out, error=self.error_from_test,
217             crashed_process_name=self._crashed_process_name,
218             crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
219
220     def _get_crash_log(self, stdout, stderr, newer_than):
221         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)
222
223     def _command_wrapper(self):
224         # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
225         if self._port.get_option('wrapper'):
226             return shlex.split(self._port.get_option('wrapper'))
227         if self._profiler:
228             return self._profiler.wrapper_arguments()
229         return []
230
231     HTTP_DIR = "http/tests/"
232     HTTP_LOCAL_DIR = "http/tests/local/"
233
234     def is_http_test(self, test_name):
235         return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)
236
237     def test_to_uri(self, test_name):
238         """Convert a test name to a URI."""
239         if not self.is_http_test(test_name):
240             return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))
241
242         relative_path = test_name[len(self.HTTP_DIR):]
243
244         # TODO(dpranke): remove the SSL reference?
245         if relative_path.startswith("ssl/"):
246             return "https://127.0.0.1:8443/" + relative_path
247         return "http://127.0.0.1:8000/" + relative_path
248
249     def uri_to_test(self, uri):
250         """Return the base layout test name for a given URI.
251
252         This returns the test name for a given URI, e.g., if you passed in
253         "file:///src/LayoutTests/fast/html/keygen.html" it would return
254         "fast/html/keygen.html".
255
256         """
257         if uri.startswith("file:///"):
258             prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
259             if not prefix.endswith('/'):
260                 prefix += '/'
261             return uri[len(prefix):]
262         if uri.startswith("http://"):
263             return uri.replace('http://127.0.0.1:8000/', self.HTTP_DIR)
264         if uri.startswith("https://"):
265             return uri.replace('https://127.0.0.1:8443/', self.HTTP_DIR)
266         raise NotImplementedError('unknown url type: %s' % uri)
267
268     def has_crashed(self):
269         if self._server_process is None:
270             return False
271         if self._crashed_process_name:
272             return True
273         if self._server_process.has_crashed():
274             self._crashed_process_name = self._server_process.name()
275             self._crashed_pid = self._server_process.pid()
276             return True
277         return False
278
279     def start(self, pixel_tests, per_test_args):
280         # FIXME: Callers shouldn't normally call this, since this routine
281         # may not be specifying the correct combination of pixel test and
282         # per_test args.
283         #
284         # The only reason we have this routine at all is so the perftestrunner
285         # can pause before running a test; it might be better to push that
286         # into run_test() directly.
287         if not self._server_process:
288             self._start(pixel_tests, per_test_args)
289             self._run_post_start_tasks()
290
291     def _setup_environ_for_driver(self, environment):
292         environment['DYLD_LIBRARY_PATH'] = self._port._build_path()
293         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
294         # FIXME: We're assuming that WebKitTestRunner checks this DumpRenderTree-named environment variable.
295         # FIXME: Commented out for now to avoid tests breaking. Re-enable after
296         # we cut over to NRWT
297         #environment['DUMPRENDERTREE_TEMP'] = str(self._port._driver_tempdir_for_environment())
298         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
299         environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir()
300         if 'WEBKIT_OUTPUTDIR' in os.environ:
301             environment['WEBKIT_OUTPUTDIR'] = os.environ['WEBKIT_OUTPUTDIR']
302         if self._profiler:
303             environment = self._profiler.adjusted_environment(environment)
304         return environment
305
306     def _start(self, pixel_tests, per_test_args):
307         self.stop()
308         self._driver_tempdir = self._port._driver_tempdir()
309         server_name = self._port.driver_name()
310         environment = self._port.setup_environ_for_server(server_name)
311         environment = self._setup_environ_for_driver(environment)
312         self._crashed_process_name = None
313         self._crashed_pid = None
314         self._server_process = self._port._server_process_constructor(self._port, server_name, self.cmd_line(pixel_tests, per_test_args), environment)
315         self._server_process.start()
316
317     def _run_post_start_tasks(self):
318         # Remote drivers may override this to delay post-start tasks until the server has ack'd.
319         if self._profiler:
320             self._profiler.attach_to_pid(self._pid_on_target())
321
322     def _pid_on_target(self):
323         # Remote drivers will override this method to return the pid on the device.
324         return self._server_process.pid()
325
326     def stop(self):
327         if self._server_process:
328             self._server_process.stop(self._port.driver_stop_timeout())
329             self._server_process = None
330             if self._profiler:
331                 self._profiler.profile_after_exit()
332
333         if self._driver_tempdir:
334             self._port._filesystem.rmtree(str(self._driver_tempdir))
335             self._driver_tempdir = None
336
337     def cmd_line(self, pixel_tests, per_test_args):
338         cmd = self._command_wrapper()
339         cmd.append(self._port._path_to_driver())
340         if self._port.get_option('gc_between_tests'):
341             cmd.append('--gc-between-tests')
342         if self._port.get_option('complex_text'):
343             cmd.append('--complex-text')
344         if self._port.get_option('accelerated_drawing'):
345             cmd.append('--accelerated-drawing')
346         if self._port.get_option('remote_layer_tree'):
347             cmd.append('--remote-layer-tree')
348         if self._port.get_option('threaded'):
349             cmd.append('--threaded')
350         if self._no_timeout:
351             cmd.append('--no-timeout')
352         # FIXME: We need to pass --timeout=SECONDS to WebKitTestRunner for WebKit2.
353
354         cmd.extend(self._port.get_option('additional_drt_flag', []))
355         cmd.extend(self._port.additional_drt_flag())
356
357         cmd.extend(per_test_args)
358
359         cmd.append('-')
360         return cmd
361
362     def _check_for_driver_crash(self, error_line):
363         if error_line == "#CRASHED\n":
364             # This is used on Windows and iOS to report that the process has crashed
365             # See http://trac.webkit.org/changeset/65537.
366             self._crashed_process_name = self._server_process.name()
367             self._crashed_pid = self._server_process.pid()
368         elif (error_line.startswith("#CRASHED - ")
369             or error_line.startswith("#PROCESS UNRESPONSIVE - ")):
370             # WebKitTestRunner/LayoutTestRelay uses this to report that the subprocess (e.g. WebProcess) crashed.
371             match = re.match('#(?:CRASHED|PROCESS UNRESPONSIVE) - (\S+)', error_line)
372             self._crashed_process_name = match.group(1) if match else 'WebProcess'
373             match = re.search('pid (\d+)', error_line)
374             pid = int(match.group(1)) if match else None
375             self._crashed_pid = pid
376             # FIXME: delete this after we're sure this code is working :)
377             _log.debug('%s crash, pid = %s, error_line = %s' % (self._crashed_process_name, str(pid), error_line))
378             if error_line.startswith("#PROCESS UNRESPONSIVE - "):
379                 self._subprocess_was_unresponsive = True
380                 self._port.sample_process(self._crashed_process_name, self._crashed_pid)
381                 # We want to show this since it's not a regular crash and probably we don't have a crash log.
382                 self.error_from_test += error_line
383             return True
384         return self.has_crashed()
385
386     def _command_from_driver_input(self, driver_input):
387         # FIXME: performance tests pass in full URLs instead of test names.
388         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
389             command = driver_input.test_name
390         elif self.is_http_test(driver_input.test_name):
391             command = self.test_to_uri(driver_input.test_name)
392         else:
393             command = self._port.abspath_for_test(driver_input.test_name)
394             if sys.platform == 'cygwin':
395                 command = path.cygpath(command)
396
397         assert not driver_input.image_hash or driver_input.should_run_pixel_test
398
399         # ' is the separator between arguments.
400         if self._port.supports_per_test_timeout():
401             command += "'--timeout'%s" % driver_input.timeout
402         if driver_input.should_run_pixel_test:
403             command += "'--pixel-test"
404         if driver_input.image_hash:
405             command += "'" + driver_input.image_hash
406         return command + "\n"
407
408     def _read_first_block(self, deadline):
409         # returns (text_content, audio_content)
410         block = self._read_block(deadline)
411         if block.malloc:
412             self._measurements['Malloc'] = float(block.malloc)
413         if block.js_heap:
414             self._measurements['JSHeap'] = float(block.js_heap)
415         if block.content_type == 'audio/wav':
416             return (None, block.decoded_content)
417         return (block.decoded_content, None)
418
419     def _read_optional_image_block(self, deadline):
420         # returns (image, actual_image_hash)
421         block = self._read_block(deadline, wait_for_stderr_eof=True)
422         if block.content and block.content_type == 'image/png':
423             return (block.decoded_content, block.content_hash)
424         return (None, block.content_hash)
425
426     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
427         if line.startswith(header_text) and getattr(block, header_attr) is None:
428             value = line.split()[1]
429             if header_filter:
430                 value = header_filter(value)
431             setattr(block, header_attr, value)
432             return True
433         return False
434
435     def _process_stdout_line(self, block, line):
436         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
437             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
438             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
439             or self._read_header(block, line, 'ActualHash: ', 'content_hash')
440             or self._read_header(block, line, 'DumpMalloc: ', 'malloc')
441             or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')):
442             return
443         # Note, we're not reading ExpectedHash: here, but we could.
444         # If the line wasn't a header, we just append it to the content.
445         block.content += line
446
447     def _strip_eof(self, line):
448         if line and line.endswith("#EOF\n"):
449             return line[:-5], True
450         return line, False
451
452     def _read_block(self, deadline, wait_for_stderr_eof=False):
453         block = ContentBlock()
454         out_seen_eof = False
455
456         while not self.has_crashed():
457             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
458                 break
459
460             if self.err_seen_eof:
461                 out_line = self._server_process.read_stdout_line(deadline)
462                 err_line = None
463             elif out_seen_eof:
464                 out_line = None
465                 err_line = self._server_process.read_stderr_line(deadline)
466             else:
467                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
468
469             if self._server_process.timed_out or self.has_crashed():
470                 break
471
472             if out_line:
473                 assert not out_seen_eof
474                 out_line, out_seen_eof = self._strip_eof(out_line)
475             if err_line:
476                 assert not self.err_seen_eof
477                 err_line, self.err_seen_eof = self._strip_eof(err_line)
478
479             if out_line:
480                 if out_line[-1] != "\n":
481                     _log.error("Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug.")
482                 content_length_before_header_check = block._content_length
483                 self._process_stdout_line(block, out_line)
484                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
485                 # Don't wait until we're done with headers, just read the binary blob right now.
486                 if content_length_before_header_check != block._content_length:
487                     block.content = self._server_process.read_stdout(deadline, block._content_length)
488
489             if err_line:
490                 if self._check_for_driver_crash(err_line):
491                     break
492                 self.error_from_test += err_line
493
494         block.decode_content()
495         return block
496
497     @staticmethod
498     def check_driver(port):
499         # This checks if the required system dependencies for the driver are met.
500         # Since this is the generic class implementation, just return True.
501         return True
502
503
504 class IOSSimulatorDriver(Driver):
505     def cmd_line(self, pixel_tests, per_test_args):
506         cmd = super(IOSSimulatorDriver, self).cmd_line(pixel_tests, per_test_args)
507         relay_tool = self._port.relay_path
508         dump_tool = cmd[0]
509         dump_tool_args = cmd[1:]
510         product_dir = self._port._build_path()
511         runtime = self._port.get_option('runtime')
512         device_type = self._port.get_option('device_type')
513         relay_args = [
514             '-runtime', runtime.identifier,
515             '-deviceType', device_type.identifier,
516             '-suffix', str(self._worker_number),
517             '-productDir', product_dir,
518             '-app', dump_tool,
519         ]
520         return [relay_tool] + relay_args + ['--'] + dump_tool_args
521
522     def _setup_environ_for_driver(self, environment):
523         environment['DEVELOPER_DIR'] = self._port.developer_dir
524         return super(IOSSimulatorDriver, self)._setup_environ_for_driver(environment)
525
526
527 class ContentBlock(object):
528     def __init__(self):
529         self.content_type = None
530         self.encoding = None
531         self.content_hash = None
532         self._content_length = None
533         # Content is treated as binary data even though the text output is usually UTF-8.
534         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
535         self.decoded_content = None
536         self.malloc = None
537         self.js_heap = None
538
539     def decode_content(self):
540         if self.encoding == 'base64' and self.content is not None:
541             self.decoded_content = base64.b64decode(self.content)
542         else:
543             self.decoded_content = self.content
544
545 class DriverProxy(object):
546     """A wrapper for managing two Driver instances, one with pixel tests and
547     one without. This allows us to handle plain text tests and ref tests with a
548     single driver."""
549
550     def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
551         self._port = port
552         self._worker_number = worker_number
553         self._driver_instance_constructor = driver_instance_constructor
554         self._no_timeout = no_timeout
555
556         # FIXME: We shouldn't need to create a driver until we actually run a test.
557         self._driver = self._make_driver(pixel_tests)
558         self._driver_cmd_line = None
559
560     def _make_driver(self, pixel_tests):
561         return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)
562
563     # FIXME: this should be a @classmethod (or implemented on Port instead).
564     def is_http_test(self, test_name):
565         return self._driver.is_http_test(test_name)
566
567     # FIXME: this should be a @classmethod (or implemented on Port instead).
568     def test_to_uri(self, test_name):
569         return self._driver.test_to_uri(test_name)
570
571     # FIXME: this should be a @classmethod (or implemented on Port instead).
572     def uri_to_test(self, uri):
573         return self._driver.uri_to_test(uri)
574
575     def run_test(self, driver_input, stop_when_done):
576         base = self._port.lookup_virtual_test_base(driver_input.test_name)
577         if base:
578             virtual_driver_input = copy.copy(driver_input)
579             virtual_driver_input.test_name = base
580             virtual_driver_input.args = self._port.lookup_virtual_test_args(driver_input.test_name)
581             return self.run_test(virtual_driver_input, stop_when_done)
582
583         pixel_tests_needed = driver_input.should_run_pixel_test
584         cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
585         if cmd_line_key != self._driver_cmd_line:
586             self._driver.stop()
587             self._driver = self._make_driver(pixel_tests_needed)
588             self._driver_cmd_line = cmd_line_key
589
590         return self._driver.run_test(driver_input, stop_when_done)
591
592     def has_crashed(self):
593         return self._driver.has_crashed()
594
595     def stop(self):
596         self._driver.stop()
597
598     # FIXME: this should be a @classmethod (or implemented on Port instead).
599     def cmd_line(self, pixel_tests=None, per_test_args=None):
600         return self._driver.cmd_line(pixel_tests, per_test_args or [])
601
602     def _cmd_line_as_key(self, pixel_tests, per_test_args):
603         return ' '.join(self.cmd_line(pixel_tests, per_test_args))