2010-09-22 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 from __future__ import with_statement
33
34 import codecs
35 import logging
36 import os
37 import re
38 import shutil
39 import signal
40 import subprocess
41 import sys
42 import tempfile
43 import time
44 import webbrowser
45
46 import base
47 import http_server
48
49 from webkitpy.common.system.executive import Executive
50 from webkitpy.layout_tests.layout_package import test_expectations
51
52 # Chromium DRT on OSX uses WebKitDriver.
53 if sys.platform == 'darwin':
54     import webkit
55
56 import websocket_server
57
58 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
59
60
61 # FIXME: This function doesn't belong in this package.
62 def check_file_exists(path_to_file, file_description, override_step=None,
63                       logging=True):
64     """Verify the file is present where expected or log an error.
65
66     Args:
67         file_name: The (human friendly) name or description of the file
68             you're looking for (e.g., "HTTP Server"). Used for error logging.
69         override_step: An optional string to be logged if the check fails.
70         logging: Whether or not log the error messages."""
71     if not os.path.exists(path_to_file):
72         if logging:
73             _log.error('Unable to find %s' % file_description)
74             _log.error('    at %s' % path_to_file)
75             if override_step:
76                 _log.error('    %s' % override_step)
77                 _log.error('')
78         return False
79     return True
80
81
82 class ChromiumPort(base.Port):
83     """Abstract base class for Chromium implementations of the Port class."""
84
85     def __init__(self, **kwargs):
86         if 'options' in kwargs:
87             options = kwargs['options']
88             if (options and (not hasattr(options, 'configuration') or
89                              options.configuration is None)):
90                 options.configuration = 'Release'
91         base.Port.__init__(self, **kwargs)
92         self._chromium_base_dir = None
93
94     def baseline_path(self):
95         return self._webkit_baseline_path(self._name)
96
97     def check_build(self, needs_http):
98         result = True
99
100         dump_render_tree_binary_path = self._path_to_driver()
101         result = check_file_exists(dump_render_tree_binary_path,
102                                     'test driver') and result
103         if result and self._options.build:
104             result = self._check_driver_build_up_to_date(
105                 self._options.configuration)
106         else:
107             _log.error('')
108
109         helper_path = self._path_to_helper()
110         if helper_path:
111             result = check_file_exists(helper_path,
112                                        'layout test helper') and result
113
114         if self._options.pixel_tests:
115             result = self.check_image_diff(
116                 'To override, invoke with --no-pixel-tests') and result
117
118         return result
119
120     def check_sys_deps(self, needs_http):
121         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
122         if self._executive.run_command(cmd, return_exit_code=True):
123             _log.error('System dependencies check failed.')
124             _log.error('To override, invoke with --nocheck-sys-deps')
125             _log.error('')
126             return False
127         return True
128
129     def check_image_diff(self, override_step=None, logging=True):
130         image_diff_path = self._path_to_image_diff()
131         return check_file_exists(image_diff_path, 'image diff exe',
132                                  override_step, logging)
133
134     def diff_image(self, expected_contents, actual_contents,
135                    diff_filename=None, tolerance=0):
136         executable = self._path_to_image_diff()
137         expected_tmpfile = tempfile.NamedTemporaryFile()
138         expected_tmpfile.write(expected_contents)
139         actual_tmpfile = tempfile.NamedTemporaryFile()
140         actual_tmpfile.write(actual_contents)
141         if diff_filename:
142             cmd = [executable, '--diff', expected_tmpfile.name,
143                    actual_tmpfile.name, diff_filename]
144         else:
145             cmd = [executable, expected_tmpfile.name, actual_tmpfile.name]
146
147         result = True
148         try:
149             if self._executive.run_command(cmd, return_exit_code=True) == 0:
150                 return False
151         except OSError, e:
152             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
153                 _compare_available = False
154             else:
155                 raise e
156         finally:
157             expected_tmpfile.close()
158             actual_tmpfile.close()
159         return result
160
161     def driver_name(self):
162         return "test_shell"
163
164     def path_from_chromium_base(self, *comps):
165         """Returns the full path to path made by joining the top of the
166         Chromium source tree and the list of path components in |*comps|."""
167         if not self._chromium_base_dir:
168             abspath = os.path.abspath(__file__)
169             offset = abspath.find('third_party')
170             if offset == -1:
171                 self._chromium_base_dir = os.path.join(
172                     abspath[0:abspath.find('WebKitTools')],
173                     'WebKit', 'chromium')
174             else:
175                 self._chromium_base_dir = abspath[0:offset]
176         return os.path.join(self._chromium_base_dir, *comps)
177
178     def path_to_test_expectations_file(self):
179         return self.path_from_webkit_base('LayoutTests', 'platform',
180             'chromium', 'test_expectations.txt')
181
182     def results_directory(self):
183         try:
184             return self.path_from_chromium_base('webkit',
185                 self._options.configuration, self._options.results_directory)
186         except AssertionError:
187             return self._build_path(self._options.configuration,
188                                     self._options.results_directory)
189
190     def setup_test_run(self):
191         # Delete the disk cache if any to ensure a clean test run.
192         dump_render_tree_binary_path = self._path_to_driver()
193         cachedir = os.path.split(dump_render_tree_binary_path)[0]
194         cachedir = os.path.join(cachedir, "cache")
195         if os.path.exists(cachedir):
196             shutil.rmtree(cachedir)
197
198     def create_driver(self, image_path, options):
199         """Starts a new Driver and returns a handle to it."""
200         if options.use_drt and sys.platform == 'darwin':
201             return webkit.WebKitDriver(self, image_path, options,
202                                        executive=self._executive)
203         return ChromiumDriver(self, image_path, options,
204                               executive=self._executive)
205
206     def start_helper(self):
207         helper_path = self._path_to_helper()
208         if helper_path:
209             _log.debug("Starting layout helper %s" % helper_path)
210             # Note: Not thread safe: http://bugs.python.org/issue2320
211             self._helper = subprocess.Popen([helper_path],
212                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
213             is_ready = self._helper.stdout.readline()
214             if not is_ready.startswith('ready'):
215                 _log.error("layout_test_helper failed to be ready")
216
217     def stop_helper(self):
218         if self._helper:
219             _log.debug("Stopping layout test helper")
220             self._helper.stdin.write("x\n")
221             self._helper.stdin.close()
222             # wait() is not threadsafe and can throw OSError due to:
223             # http://bugs.python.org/issue1731717
224             self._helper.wait()
225
226     def test_base_platform_names(self):
227         return ('linux', 'mac', 'win')
228
229     def test_expectations(self):
230         """Returns the test expectations for this port.
231
232         Basically this string should contain the equivalent of a
233         test_expectations file. See test_expectations.py for more details."""
234         expectations_path = self.path_to_test_expectations_file()
235         with codecs.open(expectations_path, "r", "utf-8") as file:
236             return file.read()
237
238     def test_expectations_overrides(self):
239         # FIXME: This drt_overrides handling should be removed when we switch
240         # from tes_shell to DRT.
241         drt_overrides = ''
242         if self._options and self._options.use_drt:
243             drt_overrides_path = self.path_from_webkit_base('LayoutTests',
244                 'platform', 'chromium', 'drt_expectations.txt')
245             if os.path.exists(drt_overrides_path):
246                 with codecs.open(drt_overrides_path, "r", "utf-8") as file:
247                     drt_overrides = file.read()
248
249         try:
250             overrides_path = self.path_from_chromium_base('webkit', 'tools',
251                 'layout_tests', 'test_expectations.txt')
252         except AssertionError:
253             return None
254         if not os.path.exists(overrides_path):
255             return None
256         with codecs.open(overrides_path, "r", "utf-8") as file:
257             return file.read() + drt_overrides
258
259     def skipped_layout_tests(self, extra_test_files=None):
260         expectations_str = self.test_expectations()
261         overrides_str = self.test_expectations_overrides()
262         test_platform_name = self.test_platform_name()
263         is_debug_mode = False
264
265         all_test_files = self.tests([])
266         if extra_test_files:
267             all_test_files.update(extra_test_files)
268
269         expectations = test_expectations.TestExpectations(
270             self, all_test_files, expectations_str, test_platform_name,
271             is_debug_mode, is_lint_mode=True, overrides=overrides_str)
272         tests_dir = self.layout_tests_dir()
273         return [self.relative_test_filename(test)
274                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
275
276     def test_platform_names(self):
277         return self.test_base_platform_names() + ('win-xp',
278             'win-vista', 'win-7')
279
280     def test_platform_name_to_name(self, test_platform_name):
281         if test_platform_name in self.test_platform_names():
282             return 'chromium-' + test_platform_name
283         raise ValueError('Unsupported test_platform_name: %s' %
284                          test_platform_name)
285
286     def test_repository_paths(self):
287         # Note: for JSON file's backward-compatibility we use 'chrome' rather
288         # than 'chromium' here.
289         repos = super(ChromiumPort, self).test_repository_paths()
290         repos.append(('chrome', self.path_from_chromium_base()))
291         return repos
292
293     #
294     # PROTECTED METHODS
295     #
296     # These routines should only be called by other methods in this file
297     # or any subclasses.
298     #
299
300     def _check_driver_build_up_to_date(self, configuration):
301         if configuration in ('Debug', 'Release'):
302             try:
303                 debug_path = self._path_to_driver('Debug')
304                 release_path = self._path_to_driver('Release')
305
306                 debug_mtime = os.stat(debug_path).st_mtime
307                 release_mtime = os.stat(release_path).st_mtime
308
309                 if (debug_mtime > release_mtime and configuration == 'Release' or
310                     release_mtime > debug_mtime and configuration == 'Debug'):
311                     _log.warning('You are not running the most '
312                                  'recent DumpRenderTree binary. You need to '
313                                  'pass --debug or not to select between '
314                                  'Debug and Release.')
315                     _log.warning('')
316             # This will fail if we don't have both a debug and release binary.
317             # That's fine because, in this case, we must already be running the
318             # most up-to-date one.
319             except OSError:
320                 pass
321         return True
322
323     def _chromium_baseline_path(self, platform):
324         if platform is None:
325             platform = self.name()
326         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
327
328     def _path_to_image_diff(self):
329         binary_name = 'image_diff'
330         if self._options.use_drt:
331             binary_name = 'ImageDiff'
332         return self._build_path(self._options.configuration, binary_name)
333
334
335 class ChromiumDriver(base.Driver):
336     """Abstract interface for test_shell."""
337
338     def __init__(self, port, image_path, options, executive=Executive()):
339         self._port = port
340         self._options = options
341         self._image_path = image_path
342         self._executive = executive
343
344     def _driver_args(self):
345         driver_args = []
346         if self._image_path:
347             driver_args.append("--pixel-tests=" + self._image_path)
348
349         if self._options.use_drt:
350             driver_args.append('--test-shell')
351         else:
352             driver_args.append('--layout-tests')
353
354         if self._options.startup_dialog:
355             driver_args.append('--testshell-startup-dialog')
356
357         if self._options.gp_fault_error_box:
358             driver_args.append('--gp-fault-error-box')
359
360         if self._options.accelerated_compositing:
361             driver_args.append('--enable-accelerated-compositing')
362
363         if self._options.accelerated_2d_canvas:
364             driver_args.append('--enable-accelerated-2d-canvas')
365         return driver_args
366
367     def start(self):
368         # FIXME: Should be an error to call this method twice.
369         cmd = self._command_wrapper(self._options.wrapper)
370         cmd.append(self._port._path_to_driver())
371         cmd += self._driver_args()
372
373         # We need to pass close_fds=True to work around Python bug #2320
374         # (otherwise we can hang when we kill DumpRenderTree when we are running
375         # multiple threads). See http://bugs.python.org/issue2320 .
376         # Note that close_fds isn't supported on Windows, but this bug only
377         # shows up on Mac and Linux.
378         close_flag = sys.platform not in ('win32', 'cygwin')
379         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
380                                       stdout=subprocess.PIPE,
381                                       stderr=subprocess.STDOUT,
382                                       close_fds=close_flag)
383
384     def poll(self):
385         # poll() is not threadsafe and can throw OSError due to:
386         # http://bugs.python.org/issue1731717
387         return self._proc.poll()
388
389     def _write_command_and_read_line(self, input=None):
390         """Returns a tuple: (line, did_crash)"""
391         try:
392             if input:
393                 if isinstance(input, unicode):
394                     # TestShell expects utf-8
395                     input = input.encode("utf-8")
396                 self._proc.stdin.write(input)
397             # DumpRenderTree text output is always UTF-8.  However some tests
398             # (e.g. webarchive) may spit out binary data instead of text so we
399             # don't bother to decode the output (for either DRT or test_shell).
400             line = self._proc.stdout.readline()
401             # We could assert() here that line correctly decodes as UTF-8.
402             return (line, False)
403         except IOError, e:
404             _log.error("IOError communicating w/ test_shell: " + str(e))
405             return (None, True)
406
407     def _test_shell_command(self, uri, timeoutms, checksum):
408         cmd = uri
409         if timeoutms:
410             cmd += ' ' + str(timeoutms)
411         if checksum:
412             cmd += ' ' + checksum
413         cmd += "\n"
414         return cmd
415
416     def run_test(self, uri, timeoutms, checksum):
417         output = []
418         error = []
419         crash = False
420         timeout = False
421         actual_uri = None
422         actual_checksum = None
423
424         start_time = time.time()
425
426         cmd = self._test_shell_command(uri, timeoutms, checksum)
427         (line, crash) = self._write_command_and_read_line(input=cmd)
428
429         while not crash and line.rstrip() != "#EOF":
430             # Make sure we haven't crashed.
431             if line == '' and self.poll() is not None:
432                 # This is hex code 0xc000001d, which is used for abrupt
433                 # termination. This happens if we hit ctrl+c from the prompt
434                 # and we happen to be waiting on test_shell.
435                 # sdoyon: Not sure for which OS and in what circumstances the
436                 # above code is valid. What works for me under Linux to detect
437                 # ctrl+c is for the subprocess returncode to be negative
438                 # SIGINT. And that agrees with the subprocess documentation.
439                 if (-1073741510 == self._proc.returncode or
440                     - signal.SIGINT == self._proc.returncode):
441                     raise KeyboardInterrupt
442                 crash = True
443                 break
444
445             # Don't include #URL lines in our output
446             if line.startswith("#URL:"):
447                 actual_uri = line.rstrip()[5:]
448                 if uri != actual_uri:
449                     # GURL capitalizes the drive letter of a file URL.
450                     if (not re.search("^file:///[a-z]:", uri) or
451                         uri.lower() != actual_uri.lower()):
452                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
453                                    (uri, actual_uri))
454                         raise AssertionError("test out of sync")
455             elif line.startswith("#MD5:"):
456                 actual_checksum = line.rstrip()[5:]
457             elif line.startswith("#TEST_TIMED_OUT"):
458                 timeout = True
459                 # Test timed out, but we still need to read until #EOF.
460             elif actual_uri:
461                 output.append(line)
462             else:
463                 error.append(line)
464
465             (line, crash) = self._write_command_and_read_line(input=None)
466
467         return (crash, timeout, actual_checksum, ''.join(output),
468                 ''.join(error))
469
470     def stop(self):
471         if self._proc:
472             self._proc.stdin.close()
473             self._proc.stdout.close()
474             if self._proc.stderr:
475                 self._proc.stderr.close()
476             if sys.platform not in ('win32', 'cygwin'):
477                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
478                 # (see __init__(), above), and anyway we don't want to hang
479                 # the harness if test_shell is buggy, so we wait a couple
480                 # seconds to give test_shell a chance to clean up, but then
481                 # force-kill the process if necessary.
482                 KILL_TIMEOUT = 3.0
483                 timeout = time.time() + KILL_TIMEOUT
484                 # poll() is not threadsafe and can throw OSError due to:
485                 # http://bugs.python.org/issue1731717
486                 while self._proc.poll() is None and time.time() < timeout:
487                     time.sleep(0.1)
488                 # poll() is not threadsafe and can throw OSError due to:
489                 # http://bugs.python.org/issue1731717
490                 if self._proc.poll() is None:
491                     _log.warning('stopping test driver timed out, '
492                                  'killing it')
493                     self._executive.kill_process(self._proc.pid)