879c79abf2b72dd0358e1bc8731c25037e0d4c36
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 # Copyright (C) 2011, 2012 Apple Inc. All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 #     * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 #     * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
15 # distribution.
16 #     * Neither the Google name nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
19 #
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32 """WebKit implementations of the Port interface."""
33
34 import base64
35 import itertools
36 import logging
37 import operator
38 import re
39 import sys
40 import time
41
42 from webkitpy.common.memoized import memoized
43 from webkitpy.common.net.buildbot import BuildBot
44 from webkitpy.common.system.environment import Environment
45 from webkitpy.common.system.executive import Executive, ScriptError
46 from webkitpy.common.system.path import cygpath
47 from webkitpy.layout_tests.port import builders, server_process, Port, Driver, DriverOutput
48
49
50 _log = logging.getLogger(__name__)
51
52
53 class WebKitPort(Port):
54     def __init__(self, host, port_name=None, **kwargs):
55         Port.__init__(self, host, port_name=port_name, **kwargs)
56
57         # FIXME: Disable pixel tests until they are run by default on build.webkit.org.
58         self.set_option_default("pixel_tests", False)
59         # WebKit ports expect a 35s timeout, or 350s timeout when running with -g/--guard-malloc.
60         # FIXME: --guard-malloc is only supported on Mac, so this logic should be in mac.py.
61         default_time_out_seconds = 350 if self.get_option('guard_malloc') else 35
62         self.set_option_default("time_out_ms", default_time_out_seconds * 1000)
63
64     def driver_name(self):
65         if self.get_option('webkit_test_runner'):
66             return "WebKitTestRunner"
67         return "DumpRenderTree"
68
69     # FIXME: Eventually we should standarize port naming, and make this method smart enough
70     # to use for all port configurations (including architectures, graphics types, etc).
71     def baseline_search_path(self):
72         search_paths = []
73         if self.get_option('webkit_test_runner'):
74             search_paths.append(self._wk2_port_name())
75         search_paths.append(self.name())
76         if self.name() != self.port_name:
77             search_paths.append(self.port_name)
78         return map(self._webkit_baseline_path, search_paths)
79
80     def path_to_test_expectations_file(self):
81         # test_expectations are always in mac/ not mac-leopard/ by convention, hence we use port_name instead of name().
82         return self._filesystem.join(self._webkit_baseline_path(self.port_name), 'test_expectations.txt')
83
84     def _port_flag_for_scripts(self):
85         # This is overrriden by ports which need a flag passed to scripts to distinguish the use of that port.
86         # For example --qt on linux, since a user might have both Gtk and Qt libraries installed.
87         # FIXME: Chromium should override this once ChromiumPort is a WebKitPort.
88         return None
89
90     # This is modeled after webkitdirs.pm argumentsForConfiguration() from old-run-webkit-tests
91     def _arguments_for_configuration(self):
92         config_args = []
93         config_args.append(self._config.flag_for_configuration(self.get_option('configuration')))
94         # FIXME: We may need to add support for passing --32-bit like old-run-webkit-tests had.
95         port_flag = self._port_flag_for_scripts()
96         if port_flag:
97             config_args.append(port_flag)
98         return config_args
99
100     def _run_script(self, script_name, args=None, include_configuration_arguments=True, decode_output=True, env=None):
101         run_script_command = [self._config.script_path(script_name)]
102         if include_configuration_arguments:
103             run_script_command.extend(self._arguments_for_configuration())
104         if args:
105             run_script_command.extend(args)
106         output = self._executive.run_command(run_script_command, cwd=self._config.webkit_base_dir(), decode_output=decode_output, env=env)
107         _log.debug('Output of %s:\n%s' % (run_script_command, output))
108         return output
109
110     def _build_driver(self):
111         environment = self.host.copy_current_environment()
112         environment.disable_gcc_smartquotes()
113         env = environment.to_dictionary()
114
115         # FIXME: We build both DumpRenderTree and WebKitTestRunner for
116         # WebKitTestRunner runs because DumpRenderTree still includes
117         # the DumpRenderTreeSupport module and the TestNetscapePlugin.
118         # These two projects should be factored out into their own
119         # projects.
120         try:
121             self._run_script("build-dumprendertree", env=env)
122             if self.get_option('webkit_test_runner'):
123                 self._run_script("build-webkittestrunner", env=env)
124         except ScriptError, e:
125             _log.error(e.message_with_output(output_limit=None))
126             return False
127         return True
128
129     def _check_driver(self):
130         driver_path = self._path_to_driver()
131         if not self._filesystem.exists(driver_path):
132             _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
133             return False
134         return True
135
136     def check_build(self, needs_http):
137         # If we're using a pre-built copy of WebKit (--root), we assume it also includes a build of DRT.
138         if not self.get_option('root') and self.get_option('build') and not self._build_driver():
139             return False
140         if not self._check_driver():
141             return False
142         if self.get_option('pixel_tests'):
143             if not self.check_image_diff():
144                 return False
145         if not self._check_port_build():
146             return False
147         return True
148
149     def _check_port_build(self):
150         # Ports can override this method to do additional checks.
151         return True
152
153     def check_image_diff(self, override_step=None, logging=True):
154         image_diff_path = self._path_to_image_diff()
155         if not self._filesystem.exists(image_diff_path):
156             _log.error("ImageDiff was not found at %s" % image_diff_path)
157             return False
158         return True
159
160     def diff_image(self, expected_contents, actual_contents, tolerance=None):
161         # Handle the case where the test didn't actually generate an image.
162         # FIXME: need unit tests for this.
163         if not actual_contents and not expected_contents:
164             return (None, 0)
165         if not actual_contents or not expected_contents:
166             # FIXME: It's not clear what we should return in this case.
167             # Maybe we should throw an exception?
168             return (True, 0)
169
170         process = self._start_image_diff_process(expected_contents, actual_contents, tolerance=tolerance)
171         return self._read_image_diff(process)
172
173     def _image_diff_command(self, tolerance=None):
174         # FIXME: There needs to be a more sane way of handling default
175         # values for options so that you can distinguish between a default
176         # value of None and a default value that wasn't set.
177         if tolerance is None:
178             if self.get_option('tolerance') is not None:
179                 tolerance = self.get_option('tolerance')
180             else:
181                 tolerance = 0.1
182
183         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
184         return command
185
186     def _start_image_diff_process(self, expected_contents, actual_contents, tolerance=None):
187         command = self._image_diff_command(tolerance)
188         environment = self.setup_environ_for_server('ImageDiff')
189         process = server_process.ServerProcess(self, 'ImageDiff', command, environment)
190
191         process.write('Content-Length: %d\n%sContent-Length: %d\n%s' % (
192             len(actual_contents), actual_contents,
193             len(expected_contents), expected_contents))
194         return process
195
196     def _read_image_diff(self, sp):
197         deadline = time.time() + 2.0
198         output = None
199         output_image = ""
200
201         while True:
202             output = sp.read_stdout_line(deadline)
203             if sp.timed_out or sp.has_crashed() or not output:
204                 break
205
206             if output.startswith('diff'):  # This is the last line ImageDiff prints.
207                 break
208
209             if output.startswith('Content-Length'):
210                 m = re.match('Content-Length: (\d+)', output)
211                 content_length = int(m.group(1))
212                 output_image = sp.read_stdout(deadline, content_length)
213                 output = sp.read_stdout_line(deadline)
214                 break
215
216         stderr = sp.pop_all_buffered_stderr()
217         if stderr:
218             _log.warn("ImageDiff produced stderr output:\n" + stderr)
219         if sp.timed_out:
220             _log.error("ImageDiff timed out")
221         if sp.has_crashed():
222             _log.error("ImageDiff crashed")
223         # FIXME: There is no need to shut down the ImageDiff server after every diff.
224         sp.stop()
225
226         diff_percent = 0
227         if output and output.startswith('diff'):
228             m = re.match('diff: (.+)% (passed|failed)', output)
229             if m.group(2) == 'passed':
230                 return [None, 0]
231             diff_percent = float(m.group(1))
232
233         return (output_image, diff_percent)
234
235     def setup_environ_for_server(self, server_name=None):
236         clean_env = super(WebKitPort, self).setup_environ_for_server(server_name)
237         self._copy_value_from_environ_if_set(clean_env, 'WEBKIT_TESTFONTS')
238         return clean_env
239
240     def default_results_directory(self):
241         # Results are store relative to the built products to make it easy
242         # to have multiple copies of webkit checked out and built.
243         return self._build_path('layout-test-results')
244
245     def _driver_class(self):
246         return WebKitDriver
247
248     def _tests_for_other_platforms(self):
249         # By default we will skip any directory under LayoutTests/platform
250         # that isn't in our baseline search path (this mirrors what
251         # old-run-webkit-tests does in findTestsToRun()).
252         # Note this returns LayoutTests/platform/*, not platform/*/*.
253         entries = self._filesystem.glob(self._webkit_baseline_path('*'))
254         dirs_to_skip = []
255         for entry in entries:
256             if self._filesystem.isdir(entry) and entry not in self.baseline_search_path():
257                 basename = self._filesystem.basename(entry)
258                 dirs_to_skip.append('platform/%s' % basename)
259         return dirs_to_skip
260
261     def _runtime_feature_list(self):
262         """If a port makes certain features available only through runtime flags, it can override this routine to indicate which ones are available."""
263         return None
264
265     def nm_command(self):
266         return 'nm'
267
268     def _webcore_symbols_string(self):
269         webcore_library_path = self._path_to_webcore_library()
270         if not webcore_library_path:
271             return None
272         try:
273             return self._executive.run_command([self.nm_command(), webcore_library_path], error_handler=Executive.ignore_error)
274         except OSError, e:
275             _log.warn("Failed to run nm: %s.  Can't determine WebCore supported features." % e)
276         return None
277
278     # Ports which use run-time feature detection should define this method and return
279     # a dictionary mapping from Feature Names to skipped directoires.  NRWT will
280     # run DumpRenderTree --print-supported-features and parse the output.
281     # If the Feature Names are not found in the output, the corresponding directories
282     # will be skipped.
283     def _missing_feature_to_skipped_tests(self):
284         """Return the supported feature dictionary. Keys are feature names and values
285         are the lists of directories to skip if the feature name is not matched."""
286         # FIXME: This list matches WebKitWin and should be moved onto the Win port.
287         return {
288             "Accelerated Compositing": ["compositing"],
289             "3D Rendering": ["animations/3d", "transforms/3d"],
290         }
291
292     # Ports which use compile-time feature detection should define this method and return
293     # a dictionary mapping from symbol substrings to possibly disabled test directories.
294     # When the symbol substrings are not matched, the directories will be skipped.
295     # If ports don't ever enable certain features, then those directories can just be
296     # in the Skipped list instead of compile-time-checked here.
297     def _missing_symbol_to_skipped_tests(self):
298         """Return the supported feature dictionary. The keys are symbol-substrings
299         and the values are the lists of directories to skip if that symbol is missing."""
300         return {
301             "MathMLElement": ["mathml"],
302             "GraphicsLayer": ["compositing"],
303             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
304             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
305             "MHTMLArchive": ["mhtml"],
306             "CSSVariableValue": ["fast/css/variables"],
307         }
308
309     def _has_test_in_directories(self, directory_lists, test_list):
310         if not test_list:
311             return False
312
313         directories = itertools.chain.from_iterable(directory_lists)
314         for directory, test in itertools.product(directories, test_list):
315             if test.startswith(directory):
316                 return True
317         return False
318
319     def _skipped_tests_for_unsupported_features(self, test_list):
320         # Only check the runtime feature list of there are tests in the test_list that might get skipped.
321         # This is a performance optimization to avoid the subprocess call to DRT.
322         if self._has_test_in_directories(self._missing_feature_to_skipped_tests().values(), test_list):
323             # If the port supports runtime feature detection, disable any tests
324             # for features missing from the runtime feature list.
325             supported_feature_list = self._runtime_feature_list()
326             # If _runtime_feature_list returns a non-None value, then prefer
327             # runtime feature detection over static feature detection.
328             if supported_feature_list is not None:
329                 return reduce(operator.add, [directories for feature, directories in self._missing_feature_to_skipped_tests().items() if feature not in supported_feature_list])
330
331         # Only check the symbols of there are tests in the test_list that might get skipped.
332         # This is a performance optimization to avoid the calling nm.
333         if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
334             # Runtime feature detection not supported, fallback to static dectection:
335             # Disable any tests for symbols missing from the webcore symbol string.
336             webcore_symbols_string = self._webcore_symbols_string()
337             if webcore_symbols_string is not None:
338                 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in webcore_symbols_string], [])
339
340         # Failed to get any runtime or symbol information, don't skip any tests.
341         return []
342
343     def _wk2_port_name(self):
344         # By current convention, the WebKit2 name is always mac-wk2, win-wk2, not mac-leopard-wk2, etc,
345         # except for Qt because WebKit2 is only supported by Qt 5.0 (therefore: qt-5.0-wk2).
346         return "%s-wk2" % self.port_name
347
348     def _skipped_file_search_paths(self):
349         # Unlike baseline_search_path, we only want to search [WK2-PORT, PORT-VERSION, PORT] and any directories
350         # included via --additional-platform-directory, not the full casade.
351         # Note order doesn't matter since the Skipped file contents are all combined.
352         search_paths = set([self.port_name])
353         if 'future' not in self.name():
354             search_paths.add(self.name())
355         if self.get_option('webkit_test_runner'):
356             # Because nearly all of the skipped tests for WebKit 2 are due to cross-platform
357             # issues, all wk2 ports share a skipped list under platform/wk2.
358             search_paths.update([self._wk2_port_name(), "wk2"])
359         search_paths.update(self.get_option("additional_platform_directory", []))
360
361         return search_paths
362
363     def test_expectations(self):
364         # This allows ports to use a combination of test_expectations.txt files and Skipped lists.
365         expectations = ''
366         expectations_path = self.path_to_test_expectations_file()
367         if self._filesystem.exists(expectations_path):
368             _log.debug("Using test_expectations.txt: %s" % expectations_path)
369             expectations = self._filesystem.read_text_file(expectations_path)
370         return expectations
371
372     def skipped_layout_tests(self, test_list):
373         tests_to_skip = set(self._expectations_from_skipped_files(self._skipped_file_search_paths()))
374         tests_to_skip.update(self._tests_for_other_platforms())
375         tests_to_skip.update(self._skipped_tests_for_unsupported_features(test_list))
376         return tests_to_skip
377
378     def _build_path(self, *comps):
379         # --root is used for running with a pre-built root (like from a nightly zip).
380         build_directory = self.get_option('root') or self.get_option('build_directory')
381         if not build_directory:
382             build_directory = self._config.build_directory(self.get_option('configuration'))
383             # Set --build-directory here Since this modifies the options object used by the worker subprocesses,
384             # it avoids the slow call out to build_directory in each subprocess.
385             self.set_option_default('build_directory', build_directory)
386         return self._filesystem.join(self._filesystem.abspath(build_directory), *comps)
387
388     def _path_to_driver(self):
389         return self._build_path(self.driver_name())
390
391     def _path_to_webcore_library(self):
392         return None
393
394     def _path_to_helper(self):
395         return None
396
397     def _path_to_image_diff(self):
398         return self._build_path('ImageDiff')
399
400     def _path_to_wdiff(self):
401         # FIXME: This does not exist on a default Mac OS X Leopard install.
402         return 'wdiff'
403
404     # FIXME: This does not belong on the port object.
405     @memoized
406     def _path_to_apache(self):
407         # The Apache binary path can vary depending on OS and distribution
408         # See http://wiki.apache.org/httpd/DistrosDefaultLayout
409         for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
410             if self._filesystem.exists(path):
411                 return path
412         _log.error("Could not find apache. Not installed or unknown path.")
413         return None
414
415     # FIXME: This belongs on some platform abstraction instead of Port.
416     def _is_redhat_based(self):
417         return self._filesystem.exists('/etc/redhat-release')
418
419     def _is_debian_based(self):
420         return self._filesystem.exists('/etc/debian_version')
421
422     # We pass sys_platform into this method to make it easy to unit test.
423     def _apache_config_file_name_for_platform(self, sys_platform):
424         if sys_platform == 'cygwin':
425             return 'cygwin-httpd.conf'  # CYGWIN is the only platform to still use Apache 1.3.
426         if sys_platform.startswith('linux'):
427             if self._is_redhat_based():
428                 return 'fedora-httpd.conf'  # This is an Apache 2.x config file despite the naming.
429             if self._is_debian_based():
430                 return 'apache2-debian-httpd.conf'
431         # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
432         return "apache2-httpd.conf"
433
434     def _path_to_apache_config_file(self):
435         config_file_name = self._apache_config_file_name_for_platform(sys.platform)
436         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
437
438
439 class WebKitDriver(Driver):
440     """WebKit implementation of the DumpRenderTree/WebKitTestRunner interface."""
441
442     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
443         Driver.__init__(self, port, worker_number, pixel_tests, no_timeout)
444         self._driver_tempdir = None
445         # WebKitTestRunner can report back subprocess crashes by printing
446         # "#CRASHED - PROCESSNAME".  Since those can happen at any time
447         # and ServerProcess won't be aware of them (since the actual tool
448         # didn't crash, just a subprocess) we record the crashed subprocess name here.
449         self._crashed_process_name = None
450         self._crashed_pid = None
451
452         # WebKitTestRunner can report back subprocesses that became unresponsive
453         # This could mean they crashed.
454         self._subprocess_was_unresponsive = False
455
456         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
457         # stderr output, as well as if we've seen #EOF on this driver instance.
458         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
459         # instead scope these locally in run_test.
460         self.error_from_test = str()
461         self.err_seen_eof = False
462         self._server_process = None
463
464     def __del__(self):
465         assert(self._server_process is None)
466         assert(self._driver_tempdir is None)
467
468     def cmd_line(self, pixel_tests, per_test_args):
469         cmd = self._command_wrapper(self._port.get_option('wrapper'))
470         cmd.append(self._port._path_to_driver())
471         if self._port.get_option('gc_between_tests'):
472             cmd.append('--gc-between-tests')
473         if self._port.get_option('complex_text'):
474             cmd.append('--complex-text')
475         if self._port.get_option('threaded'):
476             cmd.append('--threaded')
477         if self._no_timeout:
478             cmd.append('--no-timeout')
479         # FIXME: We need to pass --timeout=SECONDS to WebKitTestRunner for WebKit2.
480
481         cmd.extend(self._port.get_option('additional_drt_flag', []))
482
483         if pixel_tests:
484             cmd.append('--pixel-tests')
485         cmd.extend(per_test_args)
486
487         cmd.append('-')
488         return cmd
489
490     def _start(self, pixel_tests, per_test_args):
491         self._driver_tempdir = self._port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name())
492         server_name = self._port.driver_name()
493         environment = self._port.setup_environ_for_server(server_name)
494         environment['DYLD_LIBRARY_PATH'] = self._port._build_path()
495         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
496         # FIXME: We're assuming that WebKitTestRunner checks this DumpRenderTree-named environment variable.
497         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
498         environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir()
499         self._crashed_process_name = None
500         self._crashed_pid = None
501         self._server_process = server_process.ServerProcess(self._port, server_name, self.cmd_line(pixel_tests, per_test_args), environment)
502
503     def has_crashed(self):
504         if self._server_process is None:
505             return False
506         if self._crashed_process_name:
507             return True
508         if self._server_process.has_crashed():
509             self._crashed_process_name = self._server_process.name()
510             self._crashed_pid = self._server_process.pid()
511             return True
512         return False
513
514     def _check_for_driver_crash(self, error_line):
515         if error_line == "#CRASHED\n":
516             # This is used on Windows to report that the process has crashed
517             # See http://trac.webkit.org/changeset/65537.
518             self._crashed_process_name = self._server_process.name()
519             self._crashed_pid = self._server_process.pid()
520         elif (error_line.startswith("#CRASHED - WebProcess")
521             or error_line.startswith("#PROCESS UNRESPONSIVE - WebProcess")):
522             # WebKitTestRunner uses this to report that the WebProcess subprocess crashed.
523             pid = None
524             m = re.search('pid (\d+)', error_line)
525             if m:
526                 pid = int(m.group(1))
527             self._crashed_process_name = 'WebProcess'
528             self._crashed_pid = pid
529             # FIXME: delete this after we're sure this code is working :)
530             _log.debug('WebProcess crash, pid = %s, error_line = %s' % (str(pid), error_line))
531             if error_line.startswith("#PROCESS UNRESPONSIVE - WebProcess"):
532                 self._subprocess_was_unresponsive = True
533             return True
534         return self.has_crashed()
535
536     def _command_from_driver_input(self, driver_input):
537         # FIXME: performance tests pass in full URLs instead of test names.
538         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://'):
539             command = driver_input.test_name
540         elif self.is_http_test(driver_input.test_name):
541             command = self.test_to_uri(driver_input.test_name)
542         else:
543             command = self._port.abspath_for_test(driver_input.test_name)
544             if sys.platform == 'cygwin':
545                 command = cygpath(command)
546
547         if driver_input.image_hash:
548             # FIXME: Why the leading quote?
549             command += "'" + driver_input.image_hash
550         return command + "\n"
551
552     def _read_first_block(self, deadline):
553         # returns (text_content, audio_content)
554         block = self._read_block(deadline)
555         if block.content_type == 'audio/wav':
556             return (None, block.decoded_content)
557         return (block.decoded_content, None)
558
559     def _read_optional_image_block(self, deadline):
560         # returns (image, actual_image_hash)
561         block = self._read_block(deadline, wait_for_stderr_eof=True)
562         if block.content and block.content_type == 'image/png':
563             return (block.decoded_content, block.content_hash)
564         return (None, block.content_hash)
565
566     def run_test(self, driver_input):
567         start_time = time.time()
568         if not self._server_process:
569             self._start(driver_input.should_run_pixel_test, driver_input.args)
570         self.error_from_test = str()
571         self.err_seen_eof = False
572
573         command = self._command_from_driver_input(driver_input)
574         deadline = start_time + int(driver_input.timeout) / 1000.0
575
576         self._server_process.write(command)
577         text, audio = self._read_first_block(deadline)  # First block is either text or audio
578         image, actual_image_hash = self._read_optional_image_block(deadline)  # The second (optional) block is image data.
579
580         # We may not have read all of the output if an error (crash) occured.
581         # Since some platforms output the stacktrace over error, we should
582         # dump any buffered error into self.error_from_test.
583         # FIXME: We may need to also read stderr until the process dies?
584         self.error_from_test += self._server_process.pop_all_buffered_stderr()
585
586         crash_log = None
587         if self.has_crashed():
588             crash_log = self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, text, self.error_from_test,
589                                                   newer_than=start_time)
590
591             # If we don't find a crash log use a placeholder error message instead.
592             if not crash_log:
593                 crash_log = 'no crash log found for %s:%d.' % (self._crashed_process_name, self._crashed_pid)
594                 # If we were unresponsive append a message informing there may not have been a crash.
595                 if self._subprocess_was_unresponsive:
596                     crash_log += '  Process failed to become responsive before timing out.'
597
598         timeout = self._server_process.timed_out
599         if timeout:
600             # DRT doesn't have a built in timer to abort the test, so we might as well
601             # kill the process directly and not wait for it to shut down cleanly (since it may not).
602             self._server_process.kill()
603
604         return DriverOutput(text, image, actual_image_hash, audio,
605             crash=self.has_crashed(), test_time=time.time() - start_time,
606             timeout=timeout, error=self.error_from_test,
607             crashed_process_name=self._crashed_process_name,
608             crashed_pid=self._crashed_pid, crash_log=crash_log)
609
610     def _read_header(self, block, line, header_text, header_attr, header_filter=None):
611         if line.startswith(header_text) and getattr(block, header_attr) is None:
612             value = line.split()[1]
613             if header_filter:
614                 value = header_filter(value)
615             setattr(block, header_attr, value)
616             return True
617         return False
618
619     def _process_stdout_line(self, block, line):
620         if (self._read_header(block, line, 'Content-Type: ', 'content_type')
621             or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
622             or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
623             or self._read_header(block, line, 'ActualHash: ', 'content_hash')):
624             return
625         # Note, we're not reading ExpectedHash: here, but we could.
626         # If the line wasn't a header, we just append it to the content.
627         block.content += line
628
629     def _strip_eof(self, line):
630         if line and line.endswith("#EOF\n"):
631             return line[:-5], True
632         return line, False
633
634     def _read_block(self, deadline, wait_for_stderr_eof=False):
635         block = ContentBlock()
636         out_seen_eof = False
637
638         while not self.has_crashed():
639             if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
640                 break
641
642             if self.err_seen_eof:
643                 out_line = self._server_process.read_stdout_line(deadline)
644                 err_line = None
645             elif out_seen_eof:
646                 out_line = None
647                 err_line = self._server_process.read_stderr_line(deadline)
648             else:
649                 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)
650
651             if self._server_process.timed_out or self.has_crashed():
652                 break
653
654             if out_line:
655                 assert not out_seen_eof
656                 out_line, out_seen_eof = self._strip_eof(out_line)
657             if err_line:
658                 assert not self.err_seen_eof
659                 err_line, self.err_seen_eof = self._strip_eof(err_line)
660
661             if out_line:
662                 if out_line[-1] != "\n":
663                     _log.error("Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug.")
664                 content_length_before_header_check = block._content_length
665                 self._process_stdout_line(block, out_line)
666                 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
667                 # Don't wait until we're done with headers, just read the binary blob right now.
668                 if content_length_before_header_check != block._content_length:
669                     block.content = self._server_process.read_stdout(deadline, block._content_length)
670
671             if err_line:
672                 if self._check_for_driver_crash(err_line):
673                     break
674                 self.error_from_test += err_line
675
676         block.decode_content()
677         return block
678
679     def start(self, pixel_tests, per_test_args):
680         if not self._server_process:
681             self._start(pixel_tests, per_test_args)
682
683     def stop(self):
684         if self._server_process:
685             self._server_process.stop()
686             self._server_process = None
687
688         if self._driver_tempdir:
689             self._port._filesystem.rmtree(str(self._driver_tempdir))
690             self._driver_tempdir = None
691
692
693 class ContentBlock(object):
694     def __init__(self):
695         self.content_type = None
696         self.encoding = None
697         self.content_hash = None
698         self._content_length = None
699         # Content is treated as binary data even though the text output is usually UTF-8.
700         self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
701         self.decoded_content = None
702
703     def decode_content(self):
704         if self.encoding == 'base64':
705             self.decoded_content = base64.b64decode(self.content)
706         else:
707             self.decoded_content = self.content