2011-01-19 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 import errno
33 import logging
34 import re
35 import signal
36 import subprocess
37 import sys
38 import time
39 import webbrowser
40
41 from webkitpy.common.system import executive
42 from webkitpy.common.system.path import cygpath
43 from webkitpy.layout_tests.layout_package import test_expectations
44 from webkitpy.layout_tests.layout_package import test_output
45
46 import base
47 import http_server
48
49 # Chromium DRT on OSX uses WebKitDriver.
50 if sys.platform == 'darwin':
51     import webkit
52
53 import websocket_server
54
55 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
56
57
58 # FIXME: This function doesn't belong in this package.
59 def check_file_exists(path_to_file, file_description, override_step=None,
60                       logging=True):
61     """Verify the file is present where expected or log an error.
62
63     Args:
64         file_name: The (human friendly) name or description of the file
65             you're looking for (e.g., "HTTP Server"). Used for error logging.
66         override_step: An optional string to be logged if the check fails.
67         logging: Whether or not log the error messages."""
68     if not self._filesystem.exists(path_to_file):
69         if logging:
70             _log.error('Unable to find %s' % file_description)
71             _log.error('    at %s' % path_to_file)
72             if override_step:
73                 _log.error('    %s' % override_step)
74                 _log.error('')
75         return False
76     return True
77
78
79 class ChromiumPort(base.Port):
80     """Abstract base class for Chromium implementations of the Port class."""
81
82     def __init__(self, **kwargs):
83         base.Port.__init__(self, **kwargs)
84         self._chromium_base_dir = None
85
86     def baseline_path(self):
87         return self._webkit_baseline_path(self._name)
88
89     def check_build(self, needs_http):
90         result = True
91
92         dump_render_tree_binary_path = self._path_to_driver()
93         result = check_file_exists(dump_render_tree_binary_path,
94                                     'test driver') and result
95         if result and self.get_option('build'):
96             result = self._check_driver_build_up_to_date(
97                 self.get_option('configuration'))
98         else:
99             _log.error('')
100
101         helper_path = self._path_to_helper()
102         if helper_path:
103             result = check_file_exists(helper_path,
104                                        'layout test helper') and result
105
106         if self.get_option('pixel_tests'):
107             result = self.check_image_diff(
108                 'To override, invoke with --no-pixel-tests') and result
109
110         # It's okay if pretty patch isn't available, but we will at
111         # least log a message.
112         self.check_pretty_patch()
113
114         return result
115
116     def check_sys_deps(self, needs_http):
117         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
118
119         local_error = executive.ScriptError()
120
121         def error_handler(script_error):
122             local_error.exit_code = script_error.exit_code
123
124         output = self._executive.run_command(cmd, error_handler=error_handler)
125         if local_error.exit_code:
126             _log.error('System dependencies check failed.')
127             _log.error('To override, invoke with --nocheck-sys-deps')
128             _log.error('')
129             _log.error(output)
130             return False
131         return True
132
133     def check_image_diff(self, override_step=None, logging=True):
134         image_diff_path = self._path_to_image_diff()
135         return check_file_exists(image_diff_path, 'image diff exe',
136                                  override_step, logging)
137
138     def diff_image(self, expected_contents, actual_contents,
139                    diff_filename=None):
140         executable = self._path_to_image_diff()
141
142         tempdir = self._filesystem.mkdtemp()
143         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
144         self._filesystem.write_binary_file(expected_filename, expected_contents)
145         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
146         self._filesystem.write_binary_file(actual_filename, actual_contents)
147
148         if diff_filename:
149             cmd = [executable, '--diff', expected_filename,
150                    actual_filename, diff_filename]
151         else:
152             cmd = [executable, expected_filename, actual_filename]
153
154         result = True
155         try:
156             exit_code = self._executive.run_command(cmd, return_exit_code=True)
157             if exit_code == 0:
158                 # The images are the same.
159                 result = False
160             elif exit_code != 1:
161                 _log.error("image diff returned an exit code of "
162                            + str(exit_code))
163                 # Returning False here causes the script to think that we
164                 # successfully created the diff even though we didn't.  If
165                 # we return True, we think that the images match but the hashes
166                 # don't match.
167                 # FIXME: Figure out why image_diff returns other values.
168                 result = False
169         except OSError, e:
170             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
171                 _compare_available = False
172             else:
173                 raise e
174         finally:
175             self._filesystem.rmtree(str(tempdir))
176         return result
177
178     def driver_name(self):
179         if self._options.use_test_shell:
180             return "test_shell"
181         return "DumpRenderTree"
182
183     def path_from_chromium_base(self, *comps):
184         """Returns the full path to path made by joining the top of the
185         Chromium source tree and the list of path components in |*comps|."""
186         if not self._chromium_base_dir:
187             abspath = self._filesystem.abspath(__file__)
188             offset = abspath.find('third_party')
189             if offset == -1:
190                 self._chromium_base_dir = self._filesystem.join(
191                     abspath[0:abspath.find('Tools')],
192                     'WebKit', 'chromium')
193             else:
194                 self._chromium_base_dir = abspath[0:offset]
195         return self._filesystem.join(self._chromium_base_dir, *comps)
196
197     def path_to_test_expectations_file(self):
198         return self.path_from_webkit_base('LayoutTests', 'platform',
199             'chromium', 'test_expectations.txt')
200
201     def results_directory(self):
202         try:
203             return self.path_from_chromium_base('webkit',
204                 self.get_option('configuration'),
205                 self.get_option('results_directory'))
206         except AssertionError:
207             return self._build_path(self.get_option('configuration'),
208                                     self.get_option('results_directory'))
209
210     def setup_test_run(self):
211         # Delete the disk cache if any to ensure a clean test run.
212         dump_render_tree_binary_path = self._path_to_driver()
213         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
214         cachedir = self._filesystem.join(cachedir, "cache")
215         if self._filesystem.exists(cachedir):
216             self._filesystem.rmtree(cachedir)
217
218     def create_driver(self, worker_number):
219         """Starts a new Driver and returns a handle to it."""
220         if not self.get_option('use_test_shell') and sys.platform == 'darwin':
221             return webkit.WebKitDriver(self, worker_number)
222         return ChromiumDriver(self, worker_number)
223
224     def start_helper(self):
225         helper_path = self._path_to_helper()
226         if helper_path:
227             _log.debug("Starting layout helper %s" % helper_path)
228             # Note: Not thread safe: http://bugs.python.org/issue2320
229             self._helper = subprocess.Popen([helper_path],
230                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
231             is_ready = self._helper.stdout.readline()
232             if not is_ready.startswith('ready'):
233                 _log.error("layout_test_helper failed to be ready")
234
235     def stop_helper(self):
236         if self._helper:
237             _log.debug("Stopping layout test helper")
238             self._helper.stdin.write("x\n")
239             self._helper.stdin.close()
240             # wait() is not threadsafe and can throw OSError due to:
241             # http://bugs.python.org/issue1731717
242             self._helper.wait()
243
244     def test_base_platform_names(self):
245         return ('linux', 'mac', 'win')
246
247     def test_expectations(self):
248         """Returns the test expectations for this port.
249
250         Basically this string should contain the equivalent of a
251         test_expectations file. See test_expectations.py for more details."""
252         expectations_path = self.path_to_test_expectations_file()
253         return self._filesystem.read_text_file(expectations_path)
254
255     def test_expectations_overrides(self):
256         try:
257             overrides_path = self.path_from_chromium_base('webkit', 'tools',
258                 'layout_tests', 'test_expectations.txt')
259         except AssertionError:
260             return None
261         if not self._filesystem.exists(overrides_path):
262             return None
263         return self._filesystem.read_text_file(overrides_path)
264
265     def skipped_layout_tests(self, extra_test_files=None):
266         expectations_str = self.test_expectations()
267         overrides_str = self.test_expectations_overrides()
268         test_platform_name = self.test_platform_name()
269         is_debug_mode = False
270
271         all_test_files = self.tests([])
272         if extra_test_files:
273             all_test_files.update(extra_test_files)
274
275         expectations = test_expectations.TestExpectations(
276             self, all_test_files, expectations_str, test_platform_name,
277             is_debug_mode, is_lint_mode=False, overrides=overrides_str)
278         tests_dir = self.layout_tests_dir()
279         return [self.relative_test_filename(test)
280                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
281
282     def test_platform_names(self):
283         return self.test_base_platform_names() + ('win-xp',
284             'win-vista', 'win-7')
285
286     def test_platform_name_to_name(self, test_platform_name):
287         if test_platform_name in self.test_platform_names():
288             return 'chromium-' + test_platform_name
289         raise ValueError('Unsupported test_platform_name: %s' %
290                          test_platform_name)
291
292     def test_repository_paths(self):
293         # Note: for JSON file's backward-compatibility we use 'chrome' rather
294         # than 'chromium' here.
295         repos = super(ChromiumPort, self).test_repository_paths()
296         repos.append(('chrome', self.path_from_chromium_base()))
297         return repos
298
299     #
300     # PROTECTED METHODS
301     #
302     # These routines should only be called by other methods in this file
303     # or any subclasses.
304     #
305
306     def _check_driver_build_up_to_date(self, configuration):
307         if configuration in ('Debug', 'Release'):
308             try:
309                 debug_path = self._path_to_driver('Debug')
310                 release_path = self._path_to_driver('Release')
311
312                 debug_mtime = self._filesystem.mtime(debug_path)
313                 release_mtime = self._filesystem.mtime(release_path)
314
315                 if (debug_mtime > release_mtime and configuration == 'Release' or
316                     release_mtime > debug_mtime and configuration == 'Debug'):
317                     _log.warning('You are not running the most '
318                                  'recent DumpRenderTree binary. You need to '
319                                  'pass --debug or not to select between '
320                                  'Debug and Release.')
321                     _log.warning('')
322             # This will fail if we don't have both a debug and release binary.
323             # That's fine because, in this case, we must already be running the
324             # most up-to-date one.
325             except OSError:
326                 pass
327         return True
328
329     def _chromium_baseline_path(self, platform):
330         if platform is None:
331             platform = self.name()
332         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
333
334     def _convert_path(self, path):
335         """Handles filename conversion for subprocess command line args."""
336         # See note above in diff_image() for why we need this.
337         if sys.platform == 'cygwin':
338             return cygpath(path)
339         return path
340
341     def _path_to_image_diff(self):
342         binary_name = 'ImageDiff'
343         if self.get_option('use_test_shell'):
344             binary_name = 'image_diff'
345         return self._build_path(self.get_option('configuration'), binary_name)
346
347
348 class ChromiumDriver(base.Driver):
349     """Abstract interface for test_shell."""
350
351     def __init__(self, port, worker_number):
352         self._port = port
353         self._worker_number = worker_number
354         self._image_path = None
355         if self._port.get_option('pixel_tests'):
356             self._image_path = self._filesystem.join(
357                 self._port.get_option('results_directory'),
358                 'png_result%s.png' % self._worker_number)
359
360     def cmd_line(self):
361         cmd = self._command_wrapper(self._port.get_option('wrapper'))
362         cmd.append(self._port._path_to_driver())
363         if self._port.get_option('pixel_tests'):
364             # See note above in diff_image() for why we need _convert_path().
365             cmd.append("--pixel-tests=" +
366                        self._port._convert_path(self._image_path))
367
368         if self._port.get_option('use_test_shell'):
369             cmd.append('--layout-tests')
370         else:
371             cmd.append('--test-shell')
372
373         if self._port.get_option('startup_dialog'):
374             cmd.append('--testshell-startup-dialog')
375
376         if self._port.get_option('gp_fault_error_box'):
377             cmd.append('--gp-fault-error-box')
378
379         if self._port.get_option('js_flags') is not None:
380             cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"')
381
382         if self._port.get_option('stress_opt'):
383             cmd.append('--stress-opt')
384
385         if self._port.get_option('stress_deopt'):
386             cmd.append('--stress-deopt')
387
388         # test_shell does not support accelerated compositing.
389         if not self._port.get_option("use_test_shell"):
390             if self._port.get_option('accelerated_compositing'):
391                 cmd.append('--enable-accelerated-compositing')
392             if self._port.get_option('accelerated_2d_canvas'):
393                 cmd.append('--enable-accelerated-2d-canvas')
394             if self._port.get_option('enable_hardware_gpu'):
395                 cmd.append('--enable-hardware-gpu')
396         return cmd
397
398     def start(self):
399         # FIXME: Should be an error to call this method twice.
400         cmd = self.cmd_line()
401
402         # We need to pass close_fds=True to work around Python bug #2320
403         # (otherwise we can hang when we kill DumpRenderTree when we are running
404         # multiple threads). See http://bugs.python.org/issue2320 .
405         # Note that close_fds isn't supported on Windows, but this bug only
406         # shows up on Mac and Linux.
407         close_flag = sys.platform not in ('win32', 'cygwin')
408         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
409                                       stdout=subprocess.PIPE,
410                                       stderr=subprocess.STDOUT,
411                                       close_fds=close_flag)
412
413     def poll(self):
414         # poll() is not threadsafe and can throw OSError due to:
415         # http://bugs.python.org/issue1731717
416         return self._proc.poll()
417
418     def _write_command_and_read_line(self, input=None):
419         """Returns a tuple: (line, did_crash)"""
420         try:
421             if input:
422                 if isinstance(input, unicode):
423                     # TestShell expects utf-8
424                     input = input.encode("utf-8")
425                 self._proc.stdin.write(input)
426             # DumpRenderTree text output is always UTF-8.  However some tests
427             # (e.g. webarchive) may spit out binary data instead of text so we
428             # don't bother to decode the output (for either DRT or test_shell).
429             line = self._proc.stdout.readline()
430             # We could assert() here that line correctly decodes as UTF-8.
431             return (line, False)
432         except IOError, e:
433             _log.error("IOError communicating w/ test_shell: " + str(e))
434             return (None, True)
435
436     def _test_shell_command(self, uri, timeoutms, checksum):
437         cmd = uri
438         if timeoutms:
439             cmd += ' ' + str(timeoutms)
440         if checksum:
441             cmd += ' ' + checksum
442         cmd += "\n"
443         return cmd
444
445     def _output_image(self):
446         """Returns the image output which driver generated."""
447         png_path = self._image_path
448         if png_path and self._filesystem.isfile(png_path):
449             return self._filesystem.read_binary_file(png_path)
450         else:
451             return None
452
453     def _output_image_with_retry(self):
454         # Retry a few more times because open() sometimes fails on Windows,
455         # raising "IOError: [Errno 13] Permission denied:"
456         retry_num = 50
457         timeout_seconds = 5.0
458         for i in range(retry_num):
459             try:
460                 return self._output_image()
461             except IOError, e:
462                 if e.errno == errno.EACCES:
463                     time.sleep(timeout_seconds / retry_num)
464                 else:
465                     raise e
466         return self._output_image()
467
468     def run_test(self, test_input):
469         output = []
470         error = []
471         crash = False
472         timeout = False
473         actual_uri = None
474         actual_checksum = None
475
476         start_time = time.time()
477
478         uri = self._port.filename_to_uri(test_input.filename)
479         cmd = self._test_shell_command(uri, test_input.timeout,
480                                        test_input.image_hash)
481         (line, crash) = self._write_command_and_read_line(input=cmd)
482
483         while not crash and line.rstrip() != "#EOF":
484             # Make sure we haven't crashed.
485             if line == '' and self.poll() is not None:
486                 # This is hex code 0xc000001d, which is used for abrupt
487                 # termination. This happens if we hit ctrl+c from the prompt
488                 # and we happen to be waiting on test_shell.
489                 # sdoyon: Not sure for which OS and in what circumstances the
490                 # above code is valid. What works for me under Linux to detect
491                 # ctrl+c is for the subprocess returncode to be negative
492                 # SIGINT. And that agrees with the subprocess documentation.
493                 if (-1073741510 == self._proc.returncode or
494                     - signal.SIGINT == self._proc.returncode):
495                     raise KeyboardInterrupt
496                 crash = True
497                 break
498
499             # Don't include #URL lines in our output
500             if line.startswith("#URL:"):
501                 actual_uri = line.rstrip()[5:]
502                 if uri != actual_uri:
503                     # GURL capitalizes the drive letter of a file URL.
504                     if (not re.search("^file:///[a-z]:", uri) or
505                         uri.lower() != actual_uri.lower()):
506                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
507                                    (uri, actual_uri))
508                         raise AssertionError("test out of sync")
509             elif line.startswith("#MD5:"):
510                 actual_checksum = line.rstrip()[5:]
511             elif line.startswith("#TEST_TIMED_OUT"):
512                 timeout = True
513                 # Test timed out, but we still need to read until #EOF.
514             elif actual_uri:
515                 output.append(line)
516             else:
517                 error.append(line)
518
519             (line, crash) = self._write_command_and_read_line(input=None)
520
521         run_time = time.time() - start_time
522         return test_output.TestOutput(
523             ''.join(output), self._output_image_with_retry(), actual_checksum,
524             crash, run_time, timeout, ''.join(error))
525
526     def stop(self):
527         if self._proc:
528             self._proc.stdin.close()
529             self._proc.stdout.close()
530             if self._proc.stderr:
531                 self._proc.stderr.close()
532             if sys.platform not in ('win32', 'cygwin'):
533                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
534                 # (see __init__(), above), and anyway we don't want to hang
535                 # the harness if test_shell is buggy, so we wait a couple
536                 # seconds to give test_shell a chance to clean up, but then
537                 # force-kill the process if necessary.
538                 KILL_TIMEOUT = 3.0
539                 timeout = time.time() + KILL_TIMEOUT
540                 # poll() is not threadsafe and can throw OSError due to:
541                 # http://bugs.python.org/issue1731717
542                 while self._proc.poll() is None and time.time() < timeout:
543                     time.sleep(0.1)
544                 # poll() is not threadsafe and can throw OSError due to:
545                 # http://bugs.python.org/issue1731717
546                 if self._proc.poll() is None:
547                     _log.warning('stopping test driver timed out, '
548                                  'killing it')
549                     self._port._executive.kill_process(self._proc.pid)