[Qt][WK2] Refactor on Qt5 Layout tests' structure
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 # Copyright (C) 2011 Apple Inc. All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 #     * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 #     * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
15 # distribution.
16 #     * Neither the Google name nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
19 #
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32 """WebKit implementations of the Port interface."""
33
34 import base64
35 import itertools
36 import logging
37 import operator
38 import re
39 import sys
40 import time
41
42 from webkitpy.common.memoized import memoized
43 from webkitpy.common.net.buildbot import BuildBot
44 from webkitpy.common.system.environment import Environment
45 from webkitpy.common.system.executive import Executive, ScriptError
46 from webkitpy.common.system.path import cygpath
47 from webkitpy.layout_tests.port import builders, server_process, Port, Driver, DriverOutput
48
49
50 _log = logging.getLogger(__name__)
51
52
53 class WebKitPort(Port):
54     def __init__(self, host, port_name=None, **kwargs):
55         Port.__init__(self, host, port_name=port_name, **kwargs)
56
57         # FIXME: Disable pixel tests until they are run by default on build.webkit.org.
58         self.set_option_default("pixel_tests", False)
59         # WebKit ports expect a 35s timeout, or 350s timeout when running with -g/--guard-malloc.
60         # FIXME: --guard-malloc is only supported on Mac, so this logic should be in mac.py.
61         default_time_out_seconds = 350 if self.get_option('guard_malloc') else 35
62         self.set_option_default("time_out_ms", default_time_out_seconds * 1000)
63
64     def driver_name(self):
65         if self.get_option('webkit_test_runner'):
66             return "WebKitTestRunner"
67         return "DumpRenderTree"
68
69     # FIXME: Eventually we should standarize port naming, and make this method smart enough
70     # to use for all port configurations (including architectures, graphics types, etc).
71     def baseline_search_path(self):
72         search_paths = []
73         if self.get_option('webkit_test_runner'):
74             search_paths.append(self._wk2_port_name())
75         search_paths.append(self.name())
76         if self.name() != self.port_name:
77             search_paths.append(self.port_name)
78         return map(self._webkit_baseline_path, search_paths)
79
80     def path_to_test_expectations_file(self):
81         # test_expectations are always in mac/ not mac-leopard/ by convention, hence we use port_name instead of name().
82         return self._filesystem.join(self._webkit_baseline_path(self.port_name), 'test_expectations.txt')
83
84     def _port_flag_for_scripts(self):
85         # This is overrriden by ports which need a flag passed to scripts to distinguish the use of that port.
86         # For example --qt on linux, since a user might have both Gtk and Qt libraries installed.
87         # FIXME: Chromium should override this once ChromiumPort is a WebKitPort.
88         return None
89
90     # This is modeled after webkitdirs.pm argumentsForConfiguration() from old-run-webkit-tests
91     def _arguments_for_configuration(self):
92         config_args = []
93         config_args.append(self._config.flag_for_configuration(self.get_option('configuration')))
94         # FIXME: We may need to add support for passing --32-bit like old-run-webkit-tests had.
95         port_flag = self._port_flag_for_scripts()
96         if port_flag:
97             config_args.append(port_flag)
98         return config_args
99
100     def _run_script(self, script_name, args=None, include_configuration_arguments=True, decode_output=True, env=None):
101         run_script_command = [self._config.script_path(script_name)]
102         if include_configuration_arguments:
103             run_script_command.extend(self._arguments_for_configuration())
104         if args:
105             run_script_command.extend(args)
106         output = self._executive.run_command(run_script_command, cwd=self._config.webkit_base_dir(), decode_output=decode_output, env=env)
107         _log.debug('Output of %s:\n%s' % (run_script_command, output))
108         return output
109
110     def _build_driver(self):
111         environment = self.host.copy_current_environment()
112         environment.disable_gcc_smartquotes()
113         env = environment.to_dictionary()
114
115         # FIXME: We build both DumpRenderTree and WebKitTestRunner for
116         # WebKitTestRunner runs because DumpRenderTree still includes
117         # the DumpRenderTreeSupport module and the TestNetscapePlugin.
118         # These two projects should be factored out into their own
119         # projects.
120         try:
121             self._run_script("build-dumprendertree", env=env)
122             if self.get_option('webkit_test_runner'):
123                 self._run_script("build-webkittestrunner", env=env)
124         except ScriptError, e:
125             _log.error(e.message_with_output(output_limit=None))
126             return False
127         return True
128
129     def _check_driver(self):
130         driver_path = self._path_to_driver()
131         if not self._filesystem.exists(driver_path):
132             _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
133             return False
134         return True
135
136     def check_build(self, needs_http):
137         # If we're using a pre-built copy of WebKit (--root), we assume it also includes a build of DRT.
138         if not self.get_option('root') and self.get_option('build') and not self._build_driver():
139             return False
140         if not self._check_driver():
141             return False
142         if self.get_option('pixel_tests'):
143             if not self.check_image_diff():
144                 return False
145         if not self._check_port_build():
146             return False
147         return True
148
149     def _check_port_build(self):
150         # Ports can override this method to do additional checks.
151         return True
152
153     def check_image_diff(self, override_step=None, logging=True):
154         image_diff_path = self._path_to_image_diff()
155         if not self._filesystem.exists(image_diff_path):
156             _log.error("ImageDiff was not found at %s" % image_diff_path)
157             return False
158         return True
159
160     def diff_image(self, expected_contents, actual_contents, tolerance=None):
161         # Handle the case where the test didn't actually generate an image.
162         # FIXME: need unit tests for this.
163         if not actual_contents and not expected_contents:
164             return (None, 0)
165         if not actual_contents or not expected_contents:
166             # FIXME: It's not clear what we should return in this case.
167             # Maybe we should throw an exception?
168             return (True, 0)
169
170         process = self._start_image_diff_process(expected_contents, actual_contents, tolerance=tolerance)
171         return self._read_image_diff(process)
172
173     def _start_image_diff_process(self, expected_contents, actual_contents, tolerance=None):
174         # FIXME: There needs to be a more sane way of handling default
175         # values for options so that you can distinguish between a default
176         # value of None and a default value that wasn't set.
177         if tolerance is None:
178             if self.get_option('tolerance') is not None:
179                 tolerance = self.get_option('tolerance')
180             else:
181                 tolerance = 0.1
182         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
183         process = server_process.ServerProcess(self, 'ImageDiff', command)
184
185         process.write('Content-Length: %d\n%sContent-Length: %d\n%s' % (
186             len(actual_contents), actual_contents,
187             len(expected_contents), expected_contents))
188         return process
189
190     def _read_image_diff(self, sp):
191         deadline = time.time() + 2.0
192         output = None
193         output_image = ""
194
195         while True:
196             output = sp.read_stdout_line(deadline)
197             if sp.timed_out or sp.crashed or not output:
198                 break
199
200             if output.startswith('diff'):  # This is the last line ImageDiff prints.
201                 break
202
203             if output.startswith('Content-Length'):
204                 m = re.match('Content-Length: (\d+)', output)
205                 content_length = int(m.group(1))
206                 output_image = sp.read_stdout(deadline, content_length)
207                 output = sp.read_stdout_line(deadline)
208                 break
209
210         if sp.timed_out:
211             _log.error("ImageDiff timed out")
212         if sp.crashed:
213             _log.error("ImageDiff crashed")
214         # FIXME: There is no need to shut down the ImageDiff server after every diff.
215         sp.stop()
216
217         diff_percent = 0
218         if output and output.startswith('diff'):
219             m = re.match('diff: (.+)% (passed|failed)', output)
220             if m.group(2) == 'passed':
221                 return [None, 0]
222             diff_percent = float(m.group(1))
223
224         return (output_image, diff_percent)
225
226     def setup_environ_for_server(self, server_name=None):
227         clean_env = super(WebKitPort, self).setup_environ_for_server(server_name)
228         self._copy_value_from_environ_if_set(clean_env, 'WEBKIT_TESTFONTS')
229         return clean_env
230
231     def default_results_directory(self):
232         # Results are store relative to the built products to make it easy
233         # to have multiple copies of webkit checked out and built.
234         return self._build_path('layout-test-results')
235
236     def _driver_class(self):
237         return WebKitDriver
238
239     def _tests_for_other_platforms(self):
240         # By default we will skip any directory under LayoutTests/platform
241         # that isn't in our baseline search path (this mirrors what
242         # old-run-webkit-tests does in findTestsToRun()).
243         # Note this returns LayoutTests/platform/*, not platform/*/*.
244         entries = self._filesystem.glob(self._webkit_baseline_path('*'))
245         dirs_to_skip = []
246         for entry in entries:
247             if self._filesystem.isdir(entry) and entry not in self.baseline_search_path():
248                 basename = self._filesystem.basename(entry)
249                 dirs_to_skip.append('platform/%s' % basename)
250         return dirs_to_skip
251
252     def _runtime_feature_list(self):
253         """Return the supported features of DRT. If a port doesn't support
254         this DRT switch, it has to override this method to return None"""
255         supported_features_command = [self._path_to_driver(), '--print-supported-features']
256         try:
257             output = self._executive.run_command(supported_features_command, error_handler=Executive.ignore_error)
258         except OSError, e:
259             _log.warn("Exception running driver: %s, %s.  Driver must be built before calling WebKitPort.test_expectations()." % (supported_features_command, e))
260             return None
261
262         # Note: win/DumpRenderTree.cpp does not print a leading space before the features_string.
263         match_object = re.match("SupportedFeatures:\s*(?P<features_string>.*)\s*", output)
264         if not match_object:
265             return None
266         return match_object.group('features_string').split(' ')
267
268     def _webcore_symbols_string(self):
269         webcore_library_path = self._path_to_webcore_library()
270         if not webcore_library_path:
271             return None
272         try:
273             return self._executive.run_command(['nm', webcore_library_path], error_handler=Executive.ignore_error)
274         except OSError, e:
275             _log.warn("Failed to run nm: %s.  Can't determine WebCore supported features." % e)
276         return None
277
278     # Ports which use run-time feature detection should define this method and return
279     # a dictionary mapping from Feature Names to skipped directoires.  NRWT will
280     # run DumpRenderTree --print-supported-features and parse the output.
281     # If the Feature Names are not found in the output, the corresponding directories
282     # will be skipped.
283     def _missing_feature_to_skipped_tests(self):
284         """Return the supported feature dictionary. Keys are feature names and values
285         are the lists of directories to skip if the feature name is not matched."""
286         # FIXME: This list matches WebKitWin and should be moved onto the Win port.
287         return {
288             "Accelerated Compositing": ["compositing"],
289             "3D Rendering": ["animations/3d", "transforms/3d"],
290         }
291
292     # Ports which use compile-time feature detection should define this method and return
293     # a dictionary mapping from symbol substrings to possibly disabled test directories.
294     # When the symbol substrings are not matched, the directories will be skipped.
295     # If ports don't ever enable certain features, then those directories can just be
296     # in the Skipped list instead of compile-time-checked here.
297     def _missing_symbol_to_skipped_tests(self):
298         """Return the supported feature dictionary. The keys are symbol-substrings
299         and the values are the lists of directories to skip if that symbol is missing."""
300         return {
301             "MathMLElement": ["mathml"],
302             "GraphicsLayer": ["compositing"],
303             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
304             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
305             "MHTMLArchive": ["mhtml"],
306         }
307
308     def _has_test_in_directories(self, directory_lists, test_list):
309         if not test_list:
310             return False
311
312         directories = itertools.chain.from_iterable(directory_lists)
313         for directory, test in itertools.product(directories, test_list):
314             if test.startswith(directory):
315                 return True
316         return False
317
318     def _skipped_tests_for_unsupported_features(self, test_list):
319         # Only check the runtime feature list of there are tests in the test_list that might get skipped.
320         # This is a performance optimization to avoid the subprocess call to DRT.
321         if self._has_test_in_directories(self._missing_feature_to_skipped_tests().values(), test_list):
322             # If the port supports runtime feature detection, disable any tests
323             # for features missing from the runtime feature list.
324             supported_feature_list = self._runtime_feature_list()
325             # If _runtime_feature_list returns a non-None value, then prefer
326             # runtime feature detection over static feature detection.
327             if supported_feature_list is not None:
328                 return reduce(operator.add, [directories for feature, directories in self._missing_feature_to_skipped_tests().items() if feature not in supported_feature_list])
329
330         # Only check the symbols of there are tests in the test_list that might get skipped.
331         # This is a performance optimization to avoid the calling nm.
332         if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
333             # Runtime feature detection not supported, fallback to static dectection:
334             # Disable any tests for symbols missing from the webcore symbol string.
335             webcore_symbols_string = self._webcore_symbols_string()
336             if webcore_symbols_string is not None:
337                 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in webcore_symbols_string], [])
338
339         # Failed to get any runtime or symbol information, don't skip any tests.
340         return []
341
342     def _wk2_port_name(self):
343         # By current convention, the WebKit2 name is always mac-wk2, win-wk2, not mac-leopard-wk2, etc,
344         # except for Qt because WebKit2 is only supported by Qt 5.0 (therefore: qt-5.0-wk2).
345         return "%s-wk2" % self.port_name
346
347     def _skipped_file_search_paths(self):
348         # Unlike baseline_search_path, we only want to search [WK2-PORT, PORT-VERSION, PORT] not the full casade.
349         # Note order doesn't matter since the Skipped file contents are all combined.
350         #
351         # FIXME: It's not correct to assume that port names map directly to
352         # directory names. For example, mac-future is a port name that does
353         # not have a cooresponding directory. The WebKit2 ports are another
354         # example.
355         search_paths = set([self.port_name, self.name()])
356         if self.get_option('webkit_test_runner'):
357             # Because nearly all of the skipped tests for WebKit 2 are due to cross-platform
358             # issues, all wk2 ports share a skipped list under platform/wk2.
359             search_paths.update([self._wk2_port_name(), "wk2"])
360         return search_paths
361
362     def test_expectations(self):
363         # This allows ports to use a combination of test_expectations.txt files and Skipped lists.
364         expectations = ''
365         expectations_path = self.path_to_test_expectations_file()
366         if self._filesystem.exists(expectations_path):
367             _log.debug("Using test_expectations.txt: %s" % expectations_path)
368             expectations = self._filesystem.read_text_file(expectations_path)
369         return expectations
370
371     def skipped_layout_tests(self, test_list):
372         # Use a set to allow duplicates
373         tests_to_skip = set(self._expectations_from_skipped_files(self._skipped_file_search_paths()))
374         tests_to_skip.update(self._tests_for_other_platforms())
375         tests_to_skip.update(self._skipped_tests_for_unsupported_features(test_list))
376         return tests_to_skip
377
378     def skipped_tests(self, test_list):
379         return self.skipped_layout_tests(test_list)
380
381     def _build_path(self, *comps):
382         # --root is used for running with a pre-built root (like from a nightly zip).
383         build_directory = self.get_option('root') or self.get_option('build_directory')
384         if not build_directory:
385             build_directory = self._config.build_directory(self.get_option('configuration'))
386             # Set --build-directory here Since this modifies the options object used by the worker subprocesses,
387             # it avoids the slow call out to build_directory in each subprocess.
388             self.set_option_default('build_directory', build_directory)
389         return self._filesystem.join(build_directory, *comps)
390
391     def _path_to_driver(self):
392         return self._build_path(self.driver_name())
393
394     def _path_to_webcore_library(self):
395         return None
396
397     def _path_to_helper(self):
398         return None
399
400     def _path_to_image_diff(self):
401         return self._build_path('ImageDiff')
402
403     def _path_to_wdiff(self):
404         # FIXME: This does not exist on a default Mac OS X Leopard install.
405         return 'wdiff'
406
407     # FIXME: This does not belong on the port object.
408     @memoized
409     def _path_to_apache(self):
410         # The Apache binary path can vary depending on OS and distribution
411         # See http://wiki.apache.org/httpd/DistrosDefaultLayout
412         for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
413             if self._filesystem.exists(path):
414                 return path
415         _log.error("Could not find apache. Not installed or unknown path.")
416         return None
417
418     # FIXME: This belongs on some platform abstraction instead of Port.
419     def _is_redhat_based(self):
420         return self._filesystem.exists('/etc/redhat-release')
421
422     def _is_debian_based(self):
423         return self._filesystem.exists('/etc/debian_version')
424
425     # We pass sys_platform into this method to make it easy to unit test.
426     def _apache_config_file_name_for_platform(self, sys_platform):
427         if sys_platform == 'cygwin':
428             return 'cygwin-httpd.conf'  # CYGWIN is the only platform to still use Apache 1.3.
429         if sys_platform.startswith('linux'):
430             if self._is_redhat_based():
431                 return 'fedora-httpd.conf'  # This is an Apache 2.x config file despite the naming.
432             if self._is_debian_based():
433                 return 'apache2-debian-httpd.conf'
434         # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
435         return "apache2-httpd.conf"
436
437     def _path_to_apache_config_file(self):
438         config_file_name = self._apache_config_file_name_for_platform(sys.platform)
439         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
440
441
442 class WebKitDriver(Driver):
443     """WebKit implementation of the DumpRenderTree/WebKitTestRunner interface."""
444
445     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
446         Driver.__init__(self, port, worker_number, pixel_tests, no_timeout)
447         self._driver_tempdir = port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name())
448         # WebKitTestRunner can report back subprocess crashes by printing
449         # "#CRASHED - PROCESSNAME".  Since those can happen at any time
450         # and ServerProcess won't be aware of them (since the actual tool
451         # didn't crash, just a subprocess) we record the crashed subprocess name here.
452         self._crashed_subprocess_name = None
453
454         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
455         # stderr output, as well as if we've seen #EOF on this driver instance.
456         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
457         # instead scope these locally in run_test.
458         self.error_from_test = str()
459         self.err_seen_eof = False
460         self._server_process = None
461
462     # FIXME: This may be unsafe, as python does not guarentee any ordering of __del__ calls
463     # I believe it's possible that self._port or self._port._filesystem may already be destroyed.
464     def __del__(self):
465         self._port._filesystem.rmtree(str(self._driver_tempdir))
466
467     def cmd_line(self):
468         cmd = self._command_wrapper(self._port.get_option('wrapper'))
469         cmd.append(self._port._path_to_driver())
470         if self._port.get_option('skip_pixel_test_if_no_baseline'):
471             cmd.append('--skip-pixel-test-if-no-baseline')
472         if self._pixel_tests:
473             cmd.append('--pixel-tests')
474         if self._port.get_option('gc_between_tests'):
475             cmd.append('--gc-between-tests')
476         if self._port.get_option('complex_text'):
477             cmd.append('--complex-text')
478         if self._port.get_option('threaded'):
479             cmd.append('--threaded')
480         if self._no_timeout:
481             cmd.append('--no-timeout')
482         # FIXME: We need to pass --timeout=SECONDS to WebKitTestRunner for WebKit2.
483
484         cmd.extend(self._port.get_option('additional_drt_flag', []))
485         cmd.append('-')
486         return cmd
487
488     def _start(self):
489         server_name = self._port.driver_name()
490         environment = self._port.setup_environ_for_server(server_name)
491         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
492         # FIXME: We're assuming that WebKitTestRunner checks this DumpRenderTree-named environment variable.
493         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
494         environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir()
495         self._crashed_subprocess_name = None
496         self._server_process = server_process.ServerProcess(self._port, server_name, self.cmd_line(), environment)
497
498     def has_crashed(self):
499         if self._server_process is None:
500             return False
501         return self._server_process.poll() is not None
502
503     def _check_for_driver_crash(self, error_line):
504         if error_line == "#CRASHED\n":
505             # This is used on Windows to report that the process has crashed
506             # See http://trac.webkit.org/changeset/65537.
507             self._server_process.set_crashed(True)
508         elif error_line == "#CRASHED - WebProcess\n":
509             # WebKitTestRunner uses this to report that the WebProcess subprocess crashed.
510             self._subprocess_crashed("WebProcess")
511         return self._detected_crash()
512
513     def _detected_crash(self):
514         # We can't just check self._server_process.crashed because WebKitTestRunner
515         # can report subprocess crashes at any time by printing
516         # "#CRASHED - WebProcess", we want to count those as crashes as well.
517         return self._server_process.crashed or self._crashed_subprocess_name
518
519     def _subprocess_crashed(self, subprocess_name):
520         self._crashed_subprocess_name = subprocess_name
521
522     def _crashed_process_name(self):
523         if not self._detected_crash():
524             return None
525         return self._crashed_subprocess_name or self._server_process.process_name()
526
527     def _command_from_driver_input(self, driver_input):
528         if self.is_http_test(driver_input.test_name):
529             command = self.test_to_uri(driver_input.test_name)
530         else:
531             command = self._port.abspath_for_test(driver_input.test_name)
532             if sys.platform == 'cygwin':
533                 command = cygpath(command)
534
535         if driver_input.image_hash:
536             # FIXME: Why the leading quote?
537             command += "'" + driver_input.image_hash
538         return command + "\n"
539
540     def _read_first_block(self, deadline):
541         # returns (text_content, audio_content)
542         block = self._read_block(deadline)
543         if block.content_type == 'audio/wav':
544             return (None, block.decoded_content)
545         return (block.decoded_content, None)
546
547     def _read_optional_image_block(self, deadline):
548         # returns (image, actual_image_hash)
549         block = self._read_block(deadline, wait_for_stderr_eof=True)
550         if block.content and block.content_type == 'image/png':
551             return (block.decoded_content, block.content_hash)
552         return (None, block.content_hash)
553
554     def run_test(self, driver_input):
555         if not self._server_process:
556             self._start()
557         self.error_from_test = str()
558         self.err_seen_eof = False
559
560         command = self._command_from_driver_input(driver_input)
561         start_time = time.time()
562         deadline = time.time() + int(driver_input.timeout) / 1000.0
563
564         self._server_process.write(command)
565         text, audio = self._read_first_block(deadline)  # First block is either text or audio
566         image, actual_image_hash = self._read_optional_image_block(deadline)  # The second (optional) block is image data.
567
568         # We may not have read all of the output if an error (crash) occured.
569         # Since some platforms output the stacktrace over error, we should
570         # dump any buffered error into self.error_from_test.
571         # FIXME: We may need to also read stderr until the process dies?
572         self.error_from_test += self._server_process.pop_all_buffered_stderr()
573
574         return DriverOutput(text, image, actual_image_hash, audio,
575             crash=self._detected_crash(), test_time=time.time() - start_time,
576             timeout=self._server_process.timed_out, error=self.error_from_test,
577             crashed_process_name=self._crashed_process_name())
578
579     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
580         if line.startswith(header_text) and getattr(block, header_attr) is None:
581             value = line.split()[1]
582             if header_filter:
583                 value = header_filter(value)
584             setattr(block, header_attr, value)
585             return True
586         return False
587
588     def _process_stdout_line(self, block, line):
589         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
590             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
591             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
592             or self._read_header(block, line, 'ActualHash: ', 'content_hash')):
593             return
594         # Note, we're not reading ExpectedHash: here, but we could.
595         # If the line wasn't a header, we just append it to the content.
596         block.content += line
597
598     def _strip_eof(self, line):
599         if line and line.endswith("#EOF\n"):
600             return line[:-5], True
601         return line, False
602
603     def _read_block(self, deadline, wait_for_stderr_eof=False):
604         block = ContentBlock()
605         out_seen_eof = False
606
607         while True:
608             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
609                 break
610
611             if self.err_seen_eof:
612                 out_line = self._server_process.read_stdout_line(deadline)
613                 err_line = None
614             elif out_seen_eof:
615                 out_line = None
616                 err_line = self._server_process.read_stderr_line(deadline)
617             else:
618                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
619
620             if self._server_process.timed_out or self._detected_crash():
621                 break
622
623             if out_line:
624                 assert not out_seen_eof
625                 out_line, out_seen_eof = self._strip_eof(out_line)
626             if err_line:
627                 assert not self.err_seen_eof
628                 err_line, self.err_seen_eof = self._strip_eof(err_line)
629
630             if out_line:
631                 if out_line[-1] != "\n":
632                     _log.error("Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug.")
633                 content_length_before_header_check = block._content_length
634                 self._process_stdout_line(block, out_line)
635                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
636                 # Don't wait until we're done with headers, just read the binary blob right now.
637                 if content_length_before_header_check != block._content_length:
638                     block.content = self._server_process.read_stdout(deadline, block._content_length)
639
640             if err_line:
641                 if self._check_for_driver_crash(err_line):
642                     break
643                 self.error_from_test += err_line
644
645         block.decode_content()
646         return block
647
648     def start(self):
649         if not self._server_process:
650             self._start()
651
652     def stop(self):
653         if self._server_process:
654             self._server_process.stop()
655             self._server_process = None
656
657
658 class ContentBlock(object):
659     def __init__(self):
660         self.content_type = None
661         self.encoding = None
662         self.content_hash = None
663         self._content_length = None
664         # Content is treated as binary data even though the text output is usually UTF-8.
665         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
666         self.decoded_content = None
667
668     def decode_content(self):
669         if self.encoding == 'base64':
670             self.decoded_content = base64.b64decode(self.content)
671         else:
672             self.decoded_content = self.content