7a7731abda5c7ef5758f2f93790ff26d2c0ebb76
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 from __future__ import with_statement
33
34 import codecs
35 import logging
36 import os
37 import re
38 import shutil
39 import signal
40 import subprocess
41 import sys
42 import tempfile
43 import time
44 import webbrowser
45
46 import base
47 import http_server
48
49 from webkitpy.common.system.executive import Executive
50 from webkitpy.layout_tests.layout_package import test_expectations
51
52 # Chromium DRT on OSX uses WebKitDriver.
53 if sys.platform == 'darwin':
54     import webkit
55
56 import websocket_server
57
58 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
59
60
61 # FIXME: This function doesn't belong in this package.
62 def check_file_exists(path_to_file, file_description, override_step=None,
63                       logging=True):
64     """Verify the file is present where expected or log an error.
65
66     Args:
67         file_name: The (human friendly) name or description of the file
68             you're looking for (e.g., "HTTP Server"). Used for error logging.
69         override_step: An optional string to be logged if the check fails.
70         logging: Whether or not log the error messages."""
71     if not os.path.exists(path_to_file):
72         if logging:
73             _log.error('Unable to find %s' % file_description)
74             _log.error('    at %s' % path_to_file)
75             if override_step:
76                 _log.error('    %s' % override_step)
77                 _log.error('')
78         return False
79     return True
80
81
82 class ChromiumPort(base.Port):
83     """Abstract base class for Chromium implementations of the Port class."""
84
85     def __init__(self, **kwargs):
86         if 'options' in kwargs:
87             options = kwargs['options']
88             if (options and (not hasattr(options, 'configuration') or
89                              options.configuration is None)):
90                 options.configuration = 'Release'
91         base.Port.__init__(self, **kwargs)
92         self._chromium_base_dir = None
93
94     def baseline_path(self):
95         return self._webkit_baseline_path(self._name)
96
97     def check_build(self, needs_http):
98         result = True
99
100         dump_render_tree_binary_path = self._path_to_driver()
101         result = check_file_exists(dump_render_tree_binary_path,
102                                     'test driver') and result
103         if result and self._options.build:
104             result = self._check_driver_build_up_to_date(
105                 self._options.configuration)
106         else:
107             _log.error('')
108
109         helper_path = self._path_to_helper()
110         if helper_path:
111             result = check_file_exists(helper_path,
112                                        'layout test helper') and result
113
114         if self._options.pixel_tests:
115             result = self.check_image_diff(
116                 'To override, invoke with --no-pixel-tests') and result
117
118         return result
119
120     def check_sys_deps(self, needs_http):
121         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
122         if self._executive.run_command(cmd, return_exit_code=True):
123             _log.error('System dependencies check failed.')
124             _log.error('To override, invoke with --nocheck-sys-deps')
125             _log.error('')
126             return False
127         return True
128
129     def check_image_diff(self, override_step=None, logging=True):
130         image_diff_path = self._path_to_image_diff()
131         return check_file_exists(image_diff_path, 'image diff exe',
132                                  override_step, logging)
133
134     def diff_image(self, expected_contents, actual_contents,
135                    diff_filename=None, tolerance=0):
136         executable = self._path_to_image_diff()
137         expected_tmpfile = tempfile.NamedTemporaryFile()
138         expected_tmpfile.write(expected_contents)
139         actual_tmpfile = tempfile.NamedTemporaryFile()
140         actual_tmpfile.write(actual_contents)
141         if diff_filename:
142             cmd = [executable, '--diff', expected_tmpfile.name,
143                    actual_tmpfile.name, diff_filename]
144         else:
145             cmd = [executable, expected_tmpfile.name, actual_tmpfile.name]
146
147         result = True
148         try:
149             if self._executive.run_command(cmd, return_exit_code=True) == 0:
150                 return False
151         except OSError, e:
152             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
153                 _compare_available = False
154             else:
155                 raise e
156         finally:
157             expected_tmpfile.close()
158             actual_tmpfile.close()
159         return result
160
161     def driver_name(self):
162         return "test_shell"
163
164     def path_from_chromium_base(self, *comps):
165         """Returns the full path to path made by joining the top of the
166         Chromium source tree and the list of path components in |*comps|."""
167         if not self._chromium_base_dir:
168             abspath = os.path.abspath(__file__)
169             offset = abspath.find('third_party')
170             if offset == -1:
171                 self._chromium_base_dir = os.path.join(
172                     abspath[0:abspath.find('WebKitTools')],
173                     'WebKit', 'chromium')
174             else:
175                 self._chromium_base_dir = abspath[0:offset]
176         return os.path.join(self._chromium_base_dir, *comps)
177
178     def path_to_test_expectations_file(self):
179         return self.path_from_webkit_base('LayoutTests', 'platform',
180             'chromium', 'test_expectations.txt')
181
182     def results_directory(self):
183         try:
184             return self.path_from_chromium_base('webkit',
185                 self._options.configuration, self._options.results_directory)
186         except AssertionError:
187             return self._build_path(self._options.configuration,
188                                     self._options.results_directory)
189
190     def setup_test_run(self):
191         # Delete the disk cache if any to ensure a clean test run.
192         dump_render_tree_binary_path = self._path_to_driver()
193         cachedir = os.path.split(dump_render_tree_binary_path)[0]
194         cachedir = os.path.join(cachedir, "cache")
195         if os.path.exists(cachedir):
196             shutil.rmtree(cachedir)
197
198     def create_driver(self, image_path, options):
199         """Starts a new Driver and returns a handle to it."""
200         if options.use_drt and sys.platform == 'darwin':
201             return webkit.WebKitDriver(self, image_path, options,
202                                        executive=self._executive)
203         return ChromiumDriver(self, image_path, options,
204                               executive=self._executive)
205
206     def start_helper(self):
207         helper_path = self._path_to_helper()
208         if helper_path:
209             _log.debug("Starting layout helper %s" % helper_path)
210             # Note: Not thread safe: http://bugs.python.org/issue2320
211             self._helper = subprocess.Popen([helper_path],
212                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
213             is_ready = self._helper.stdout.readline()
214             if not is_ready.startswith('ready'):
215                 _log.error("layout_test_helper failed to be ready")
216
217     def stop_helper(self):
218         if self._helper:
219             _log.debug("Stopping layout test helper")
220             self._helper.stdin.write("x\n")
221             self._helper.stdin.close()
222             # wait() is not threadsafe and can throw OSError due to:
223             # http://bugs.python.org/issue1731717
224             self._helper.wait()
225
226     def test_base_platform_names(self):
227         return ('linux', 'mac', 'win')
228
229     def test_expectations(self):
230         """Returns the test expectations for this port.
231
232         Basically this string should contain the equivalent of a
233         test_expectations file. See test_expectations.py for more details."""
234         expectations_path = self.path_to_test_expectations_file()
235         with codecs.open(expectations_path, "r", "utf-8") as file:
236             return file.read()
237
238     def test_expectations_overrides(self):
239         # FIXME: This drt_overrides handling should be removed when we switch
240         # from tes_shell to DRT.
241         drt_overrides = ''
242         if self._options.use_drt:
243             drt_overrides_path = self.path_from_webkit_base('LayoutTests',
244                 'platform', 'chromium', 'drt_expectations.txt')
245             if os.path.exists(drt_overrides_path):
246                 with codecs.open(drt_overrides_path, "r", "utf-8") as file:
247                     drt_overrides = file.read()
248
249         try:
250             overrides_path = self.path_from_chromium_base('webkit', 'tools',
251                 'layout_tests', 'test_expectations.txt')
252         except AssertionError:
253             return None
254         if not os.path.exists(overrides_path):
255             return None
256         with codecs.open(overrides_path, "r", "utf-8") as file:
257             return file.read() + drt_overrides
258
259     def skipped_layout_tests(self, extra_test_files=None):
260         expectations_str = self.test_expectations()
261         overrides_str = self.test_expectations_overrides()
262         test_platform_name = self.test_platform_name()
263         is_debug_mode = False
264
265         all_test_files = self.tests([])
266         if extra_test_files:
267             all_test_files.update(extra_test_files)
268
269         expectations = test_expectations.TestExpectations(
270             self, all_test_files, expectations_str, test_platform_name,
271             is_debug_mode, is_lint_mode=True, overrides=overrides_str)
272         tests_dir = self.layout_tests_dir()
273         return [self.relative_test_filename(test)
274                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
275
276     def test_platform_names(self):
277         return self.test_base_platform_names() + ('win-xp',
278             'win-vista', 'win-7')
279
280     def test_platform_name_to_name(self, test_platform_name):
281         if test_platform_name in self.test_platform_names():
282             return 'chromium-' + test_platform_name
283         raise ValueError('Unsupported test_platform_name: %s' %
284                          test_platform_name)
285
286     def test_repository_paths(self):
287         # Note: for JSON file's backward-compatibility we use 'chrome' rather
288         # than 'chromium' here.
289         repos = super(ChromiumPort, self).test_repository_paths()
290         repos.append(('chrome', self.path_from_chromium_base()))
291         return repos
292
293     #
294     # PROTECTED METHODS
295     #
296     # These routines should only be called by other methods in this file
297     # or any subclasses.
298     #
299
300     def _check_driver_build_up_to_date(self, configuration):
301         if configuration in ('Debug', 'Release'):
302             try:
303                 debug_path = self._path_to_driver('Debug')
304                 release_path = self._path_to_driver('Release')
305
306                 debug_mtime = os.stat(debug_path).st_mtime
307                 release_mtime = os.stat(release_path).st_mtime
308
309                 if (debug_mtime > release_mtime and configuration == 'Release' or
310                     release_mtime > debug_mtime and configuration == 'Debug'):
311                     _log.warning('You are not running the most '
312                                  'recent DumpRenderTree binary. You need to '
313                                  'pass --debug or not to select between '
314                                  'Debug and Release.')
315                     _log.warning('')
316             # This will fail if we don't have both a debug and release binary.
317             # That's fine because, in this case, we must already be running the
318             # most up-to-date one.
319             except OSError:
320                 pass
321         return True
322
323     def _chromium_baseline_path(self, platform):
324         if platform is None:
325             platform = self.name()
326         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
327
328     def _path_to_image_diff(self):
329         binary_name = 'image_diff'
330         if self._options.use_drt:
331             binary_name = 'ImageDiff'
332         return self._build_path(self._options.configuration, binary_name)
333
334
335 class ChromiumDriver(base.Driver):
336     """Abstract interface for test_shell."""
337
338     def __init__(self, port, image_path, options, executive=Executive()):
339         self._port = port
340         self._options = options
341         self._image_path = image_path
342         self._executive = executive
343
344     def _driver_args(self):
345         driver_args = []
346         if self._image_path:
347             driver_args.append("--pixel-tests=" + self._image_path)
348
349         if self._options.use_drt:
350             driver_args.append('--test-shell')
351         else:
352             driver_args.append('--layout-tests')
353
354         if self._options.startup_dialog:
355             driver_args.append('--testshell-startup-dialog')
356
357         if self._options.gp_fault_error_box:
358             driver_args.append('--gp-fault-error-box')
359         return driver_args
360
361     def start(self):
362         # FIXME: Should be an error to call this method twice.
363         cmd = self._command_wrapper(self._options.wrapper)
364         cmd.append(self._port._path_to_driver())
365         cmd += self._driver_args()
366
367         # We need to pass close_fds=True to work around Python bug #2320
368         # (otherwise we can hang when we kill DumpRenderTree when we are running
369         # multiple threads). See http://bugs.python.org/issue2320 .
370         # Note that close_fds isn't supported on Windows, but this bug only
371         # shows up on Mac and Linux.
372         close_flag = sys.platform not in ('win32', 'cygwin')
373         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
374                                       stdout=subprocess.PIPE,
375                                       stderr=subprocess.STDOUT,
376                                       close_fds=close_flag)
377
378     def poll(self):
379         # poll() is not threadsafe and can throw OSError due to:
380         # http://bugs.python.org/issue1731717
381         return self._proc.poll()
382
383     def _write_command_and_read_line(self, input=None):
384         """Returns a tuple: (line, did_crash)"""
385         try:
386             if input:
387                 if isinstance(input, unicode):
388                     # TestShell expects utf-8
389                     input = input.encode("utf-8")
390                 self._proc.stdin.write(input)
391             # DumpRenderTree text output is always UTF-8.  However some tests
392             # (e.g. webarchive) may spit out binary data instead of text so we
393             # don't bother to decode the output (for either DRT or test_shell).
394             line = self._proc.stdout.readline()
395             # We could assert() here that line correctly decodes as UTF-8.
396             return (line, False)
397         except IOError, e:
398             _log.error("IOError communicating w/ test_shell: " + str(e))
399             return (None, True)
400
401     def _test_shell_command(self, uri, timeoutms, checksum):
402         cmd = uri
403         if timeoutms:
404             cmd += ' ' + str(timeoutms)
405         if checksum:
406             cmd += ' ' + checksum
407         cmd += "\n"
408         return cmd
409
410     def run_test(self, uri, timeoutms, checksum):
411         output = []
412         error = []
413         crash = False
414         timeout = False
415         actual_uri = None
416         actual_checksum = None
417
418         start_time = time.time()
419
420         cmd = self._test_shell_command(uri, timeoutms, checksum)
421         (line, crash) = self._write_command_and_read_line(input=cmd)
422
423         while not crash and line.rstrip() != "#EOF":
424             # Make sure we haven't crashed.
425             if line == '' and self.poll() is not None:
426                 # This is hex code 0xc000001d, which is used for abrupt
427                 # termination. This happens if we hit ctrl+c from the prompt
428                 # and we happen to be waiting on test_shell.
429                 # sdoyon: Not sure for which OS and in what circumstances the
430                 # above code is valid. What works for me under Linux to detect
431                 # ctrl+c is for the subprocess returncode to be negative
432                 # SIGINT. And that agrees with the subprocess documentation.
433                 if (-1073741510 == self._proc.returncode or
434                     - signal.SIGINT == self._proc.returncode):
435                     raise KeyboardInterrupt
436                 crash = True
437                 break
438
439             # Don't include #URL lines in our output
440             if line.startswith("#URL:"):
441                 actual_uri = line.rstrip()[5:]
442                 if uri != actual_uri:
443                     # GURL capitalizes the drive letter of a file URL.
444                     if (not re.search("^file:///[a-z]:", uri) or
445                         uri.lower() != actual_uri.lower()):
446                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
447                                    (uri, actual_uri))
448                         raise AssertionError("test out of sync")
449             elif line.startswith("#MD5:"):
450                 actual_checksum = line.rstrip()[5:]
451             elif line.startswith("#TEST_TIMED_OUT"):
452                 timeout = True
453                 # Test timed out, but we still need to read until #EOF.
454             elif actual_uri:
455                 output.append(line)
456             else:
457                 error.append(line)
458
459             (line, crash) = self._write_command_and_read_line(input=None)
460
461         return (crash, timeout, actual_checksum, ''.join(output),
462                 ''.join(error))
463
464     def stop(self):
465         if self._proc:
466             self._proc.stdin.close()
467             self._proc.stdout.close()
468             if self._proc.stderr:
469                 self._proc.stderr.close()
470             if sys.platform not in ('win32', 'cygwin'):
471                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
472                 # (see __init__(), above), and anyway we don't want to hang
473                 # the harness if test_shell is buggy, so we wait a couple
474                 # seconds to give test_shell a chance to clean up, but then
475                 # force-kill the process if necessary.
476                 KILL_TIMEOUT = 3.0
477                 timeout = time.time() + KILL_TIMEOUT
478                 # poll() is not threadsafe and can throw OSError due to:
479                 # http://bugs.python.org/issue1731717
480                 while self._proc.poll() is None and time.time() < timeout:
481                     time.sleep(0.1)
482                 # poll() is not threadsafe and can throw OSError due to:
483                 # http://bugs.python.org/issue1731717
484                 if self._proc.poll() is None:
485                     _log.warning('stopping test driver timed out, '
486                                  'killing it')
487                     self._executive.kill_process(self._proc.pid)