2010-11-17 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 from __future__ import with_statement
33
34 import codecs
35 import logging
36 import os
37 import re
38 import shutil
39 import signal
40 import subprocess
41 import sys
42 import tempfile
43 import time
44 import webbrowser
45
46 from webkitpy.common.system.executive import Executive
47 from webkitpy.common.system.path import cygpath
48 from webkitpy.layout_tests.layout_package import test_expectations
49 from webkitpy.layout_tests.layout_package import test_output
50
51 import base
52 import http_server
53
54 # Chromium DRT on OSX uses WebKitDriver.
55 if sys.platform == 'darwin':
56     import webkit
57
58 import websocket_server
59
60 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
61
62
63 # FIXME: This function doesn't belong in this package.
64 def check_file_exists(path_to_file, file_description, override_step=None,
65                       logging=True):
66     """Verify the file is present where expected or log an error.
67
68     Args:
69         file_name: The (human friendly) name or description of the file
70             you're looking for (e.g., "HTTP Server"). Used for error logging.
71         override_step: An optional string to be logged if the check fails.
72         logging: Whether or not log the error messages."""
73     if not os.path.exists(path_to_file):
74         if logging:
75             _log.error('Unable to find %s' % file_description)
76             _log.error('    at %s' % path_to_file)
77             if override_step:
78                 _log.error('    %s' % override_step)
79                 _log.error('')
80         return False
81     return True
82
83
84 class ChromiumPort(base.Port):
85     """Abstract base class for Chromium implementations of the Port class."""
86
87     def __init__(self, **kwargs):
88         base.Port.__init__(self, **kwargs)
89         self._chromium_base_dir = None
90
91     def baseline_path(self):
92         return self._webkit_baseline_path(self._name)
93
94     def check_build(self, needs_http):
95         result = True
96
97         dump_render_tree_binary_path = self._path_to_driver()
98         result = check_file_exists(dump_render_tree_binary_path,
99                                     'test driver') and result
100         if result and self.get_option('build'):
101             result = self._check_driver_build_up_to_date(
102                 self.get_option('configuration'))
103         else:
104             _log.error('')
105
106         helper_path = self._path_to_helper()
107         if helper_path:
108             result = check_file_exists(helper_path,
109                                        'layout test helper') and result
110
111         if self.get_option('pixel_tests'):
112             result = self.check_image_diff(
113                 'To override, invoke with --no-pixel-tests') and result
114
115         # It's okay if pretty patch isn't available, but we will at
116         # least log a message.
117         self.check_pretty_patch()
118
119         return result
120
121     def check_sys_deps(self, needs_http):
122         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
123         if self._executive.run_command(cmd, return_exit_code=True):
124             _log.error('System dependencies check failed.')
125             _log.error('To override, invoke with --nocheck-sys-deps')
126             _log.error('')
127             return False
128         return True
129
130     def check_image_diff(self, override_step=None, logging=True):
131         image_diff_path = self._path_to_image_diff()
132         return check_file_exists(image_diff_path, 'image diff exe',
133                                  override_step, logging)
134
135     def diff_image(self, expected_contents, actual_contents,
136                    diff_filename=None):
137         executable = self._path_to_image_diff()
138
139         tempdir = tempfile.mkdtemp()
140         expected_filename = os.path.join(tempdir, "expected.png")
141         with open(expected_filename, 'w+b') as file:
142             file.write(expected_contents)
143         actual_filename = os.path.join(tempdir, "actual.png")
144         with open(actual_filename, 'w+b') as file:
145             file.write(actual_contents)
146
147         if diff_filename:
148             cmd = [executable, '--diff', expected_filename,
149                    actual_filename, diff_filename]
150         else:
151             cmd = [executable, expected_filename, actual_filename]
152
153         result = True
154         try:
155             exit_code = self._executive.run_command(cmd, return_exit_code=True)
156             if exit_code == 0:
157                 # The images are the same.
158                 result = False
159             elif exit_code != 1:
160                 _log.error("image diff returned an exit code of "
161                            + str(exit_code))
162                 # Returning False here causes the script to think that we
163                 # successfully created the diff even though we didn't.  If
164                 # we return True, we think that the images match but the hashes
165                 # don't match.
166                 # FIXME: Figure out why image_diff returns other values.
167                 result = False
168         except OSError, e:
169             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
170                 _compare_available = False
171             else:
172                 raise e
173         finally:
174             shutil.rmtree(tempdir, ignore_errors=True)
175         return result
176
177     def driver_name(self):
178         return "test_shell"
179
180     def path_from_chromium_base(self, *comps):
181         """Returns the full path to path made by joining the top of the
182         Chromium source tree and the list of path components in |*comps|."""
183         if not self._chromium_base_dir:
184             abspath = os.path.abspath(__file__)
185             offset = abspath.find('third_party')
186             if offset == -1:
187                 self._chromium_base_dir = os.path.join(
188                     abspath[0:abspath.find('WebKitTools')],
189                     'WebKit', 'chromium')
190             else:
191                 self._chromium_base_dir = abspath[0:offset]
192         return os.path.join(self._chromium_base_dir, *comps)
193
194     def path_to_test_expectations_file(self):
195         return self.path_from_webkit_base('LayoutTests', 'platform',
196             'chromium', 'test_expectations.txt')
197
198     def results_directory(self):
199         try:
200             return self.path_from_chromium_base('webkit',
201                 self.get_option('configuration'),
202                 self.get_option('results_directory'))
203         except AssertionError:
204             return self._build_path(self.get_option('configuration'),
205                                     self.get_option('results_directory'))
206
207     def setup_test_run(self):
208         # Delete the disk cache if any to ensure a clean test run.
209         dump_render_tree_binary_path = self._path_to_driver()
210         cachedir = os.path.split(dump_render_tree_binary_path)[0]
211         cachedir = os.path.join(cachedir, "cache")
212         if os.path.exists(cachedir):
213             shutil.rmtree(cachedir)
214
215     def create_driver(self, image_path, options):
216         """Starts a new Driver and returns a handle to it."""
217         if options.use_drt and sys.platform == 'darwin':
218             return webkit.WebKitDriver(self, image_path, options,
219                                        executive=self._executive)
220         return ChromiumDriver(self, image_path, options,
221                               executive=self._executive)
222
223     def start_helper(self):
224         helper_path = self._path_to_helper()
225         if helper_path:
226             _log.debug("Starting layout helper %s" % helper_path)
227             # Note: Not thread safe: http://bugs.python.org/issue2320
228             self._helper = subprocess.Popen([helper_path],
229                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
230             is_ready = self._helper.stdout.readline()
231             if not is_ready.startswith('ready'):
232                 _log.error("layout_test_helper failed to be ready")
233
234     def stop_helper(self):
235         if self._helper:
236             _log.debug("Stopping layout test helper")
237             self._helper.stdin.write("x\n")
238             self._helper.stdin.close()
239             # wait() is not threadsafe and can throw OSError due to:
240             # http://bugs.python.org/issue1731717
241             self._helper.wait()
242
243     def test_base_platform_names(self):
244         return ('linux', 'mac', 'win')
245
246     def test_expectations(self):
247         """Returns the test expectations for this port.
248
249         Basically this string should contain the equivalent of a
250         test_expectations file. See test_expectations.py for more details."""
251         expectations_path = self.path_to_test_expectations_file()
252         with codecs.open(expectations_path, "r", "utf-8") as file:
253             return file.read()
254
255     def test_expectations_overrides(self):
256         # FIXME: This drt_overrides handling should be removed when we switch
257         # from tes_shell to DRT.
258         drt_overrides = ''
259         if self.get_option('use_drt'):
260             drt_overrides_path = self.path_from_webkit_base('LayoutTests',
261                 'platform', 'chromium', 'drt_expectations.txt')
262             if os.path.exists(drt_overrides_path):
263                 with codecs.open(drt_overrides_path, "r", "utf-8") as file:
264                     drt_overrides = file.read()
265
266         try:
267             overrides_path = self.path_from_chromium_base('webkit', 'tools',
268                 'layout_tests', 'test_expectations.txt')
269         except AssertionError:
270             return None
271         if not os.path.exists(overrides_path):
272             return None
273         with codecs.open(overrides_path, "r", "utf-8") as file:
274             return file.read() + drt_overrides
275
276     def skipped_layout_tests(self, extra_test_files=None):
277         expectations_str = self.test_expectations()
278         overrides_str = self.test_expectations_overrides()
279         test_platform_name = self.test_platform_name()
280         is_debug_mode = False
281
282         all_test_files = self.tests([])
283         if extra_test_files:
284             all_test_files.update(extra_test_files)
285
286         expectations = test_expectations.TestExpectations(
287             self, all_test_files, expectations_str, test_platform_name,
288             is_debug_mode, is_lint_mode=True, overrides=overrides_str)
289         tests_dir = self.layout_tests_dir()
290         return [self.relative_test_filename(test)
291                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
292
293     def test_platform_names(self):
294         return self.test_base_platform_names() + ('win-xp',
295             'win-vista', 'win-7')
296
297     def test_platform_name_to_name(self, test_platform_name):
298         if test_platform_name in self.test_platform_names():
299             return 'chromium-' + test_platform_name
300         raise ValueError('Unsupported test_platform_name: %s' %
301                          test_platform_name)
302
303     def test_repository_paths(self):
304         # Note: for JSON file's backward-compatibility we use 'chrome' rather
305         # than 'chromium' here.
306         repos = super(ChromiumPort, self).test_repository_paths()
307         repos.append(('chrome', self.path_from_chromium_base()))
308         return repos
309
310     #
311     # PROTECTED METHODS
312     #
313     # These routines should only be called by other methods in this file
314     # or any subclasses.
315     #
316
317     def _check_driver_build_up_to_date(self, configuration):
318         if configuration in ('Debug', 'Release'):
319             try:
320                 debug_path = self._path_to_driver('Debug')
321                 release_path = self._path_to_driver('Release')
322
323                 debug_mtime = os.stat(debug_path).st_mtime
324                 release_mtime = os.stat(release_path).st_mtime
325
326                 if (debug_mtime > release_mtime and configuration == 'Release' or
327                     release_mtime > debug_mtime and configuration == 'Debug'):
328                     _log.warning('You are not running the most '
329                                  'recent DumpRenderTree binary. You need to '
330                                  'pass --debug or not to select between '
331                                  'Debug and Release.')
332                     _log.warning('')
333             # This will fail if we don't have both a debug and release binary.
334             # That's fine because, in this case, we must already be running the
335             # most up-to-date one.
336             except OSError:
337                 pass
338         return True
339
340     def _chromium_baseline_path(self, platform):
341         if platform is None:
342             platform = self.name()
343         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
344
345     def _convert_path(self, path):
346         """Handles filename conversion for subprocess command line args."""
347         # See note above in diff_image() for why we need this.
348         if sys.platform == 'cygwin':
349             return cygpath(path)
350         return path
351
352     def _path_to_image_diff(self):
353         binary_name = 'image_diff'
354         if self.get_option('use_drt'):
355             binary_name = 'ImageDiff'
356         return self._build_path(self.get_option('configuration'), binary_name)
357
358
359 class ChromiumDriver(base.Driver):
360     """Abstract interface for test_shell."""
361
362     def __init__(self, port, image_path, options, executive=Executive()):
363         self._port = port
364         self._options = options
365         self._image_path = image_path
366         self._executive = executive
367
368     def _driver_args(self):
369         driver_args = []
370         if self._image_path:
371             # See note above in diff_image() for why we need _convert_path().
372             driver_args.append("--pixel-tests=" +
373                                self._port._convert_path(self._image_path))
374
375         if self._port.get_option('use_drt'):
376             driver_args.append('--test-shell')
377         else:
378             driver_args.append('--layout-tests')
379
380         if self._port.get_option('startup_dialog'):
381             driver_args.append('--testshell-startup-dialog')
382
383         if self._port.get_option('gp_fault_error_box'):
384             driver_args.append('--gp-fault-error-box')
385
386         if self._options.js_flags is not None:
387             driver_args.append('--js-flags="' + self._options.js_flags + '"')
388
389         if self._options.multiple_loads is not None and self._options.multiple_loads > 0:
390             driver_args.append('--multiple-loads=' + str(self._options.multiple_loads))
391
392         if self._port.get_option('accelerated_compositing'):
393             driver_args.append('--enable-accelerated-compositing')
394
395         if self._port.get_option('accelerated_2d_canvas'):
396             driver_args.append('--enable-accelerated-2d-canvas')
397         return driver_args
398
399     def start(self):
400         # FIXME: Should be an error to call this method twice.
401         cmd = self._command_wrapper(self._port.get_option('wrapper'))
402         cmd.append(self._port._path_to_driver())
403         cmd += self._driver_args()
404
405         # We need to pass close_fds=True to work around Python bug #2320
406         # (otherwise we can hang when we kill DumpRenderTree when we are running
407         # multiple threads). See http://bugs.python.org/issue2320 .
408         # Note that close_fds isn't supported on Windows, but this bug only
409         # shows up on Mac and Linux.
410         close_flag = sys.platform not in ('win32', 'cygwin')
411         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
412                                       stdout=subprocess.PIPE,
413                                       stderr=subprocess.STDOUT,
414                                       close_fds=close_flag)
415
416     def poll(self):
417         # poll() is not threadsafe and can throw OSError due to:
418         # http://bugs.python.org/issue1731717
419         return self._proc.poll()
420
421     def _write_command_and_read_line(self, input=None):
422         """Returns a tuple: (line, did_crash)"""
423         try:
424             if input:
425                 if isinstance(input, unicode):
426                     # TestShell expects utf-8
427                     input = input.encode("utf-8")
428                 self._proc.stdin.write(input)
429             # DumpRenderTree text output is always UTF-8.  However some tests
430             # (e.g. webarchive) may spit out binary data instead of text so we
431             # don't bother to decode the output (for either DRT or test_shell).
432             line = self._proc.stdout.readline()
433             # We could assert() here that line correctly decodes as UTF-8.
434             return (line, False)
435         except IOError, e:
436             _log.error("IOError communicating w/ test_shell: " + str(e))
437             return (None, True)
438
439     def _test_shell_command(self, uri, timeoutms, checksum):
440         cmd = uri
441         if timeoutms:
442             cmd += ' ' + str(timeoutms)
443         if checksum:
444             cmd += ' ' + checksum
445         cmd += "\n"
446         return cmd
447
448     def _output_image(self):
449         """Returns the image output which driver generated."""
450         png_path = self._image_path
451         if png_path and os.path.isfile(png_path):
452             with open(png_path, 'rb') as image_file:
453                 return image_file.read()
454         else:
455             return None
456
457     def run_test(self, uri, timeoutms, checksum):
458         output = []
459         error = []
460         crash = False
461         timeout = False
462         actual_uri = None
463         actual_checksum = None
464
465         start_time = time.time()
466
467         cmd = self._test_shell_command(uri, timeoutms, checksum)
468         (line, crash) = self._write_command_and_read_line(input=cmd)
469
470         while not crash and line.rstrip() != "#EOF":
471             # Make sure we haven't crashed.
472             if line == '' and self.poll() is not None:
473                 # This is hex code 0xc000001d, which is used for abrupt
474                 # termination. This happens if we hit ctrl+c from the prompt
475                 # and we happen to be waiting on test_shell.
476                 # sdoyon: Not sure for which OS and in what circumstances the
477                 # above code is valid. What works for me under Linux to detect
478                 # ctrl+c is for the subprocess returncode to be negative
479                 # SIGINT. And that agrees with the subprocess documentation.
480                 if (-1073741510 == self._proc.returncode or
481                     - signal.SIGINT == self._proc.returncode):
482                     raise KeyboardInterrupt
483                 crash = True
484                 break
485
486             # Don't include #URL lines in our output
487             if line.startswith("#URL:"):
488                 actual_uri = line.rstrip()[5:]
489                 if uri != actual_uri:
490                     # GURL capitalizes the drive letter of a file URL.
491                     if (not re.search("^file:///[a-z]:", uri) or
492                         uri.lower() != actual_uri.lower()):
493                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
494                                    (uri, actual_uri))
495                         raise AssertionError("test out of sync")
496             elif line.startswith("#MD5:"):
497                 actual_checksum = line.rstrip()[5:]
498             elif line.startswith("#TEST_TIMED_OUT"):
499                 timeout = True
500                 # Test timed out, but we still need to read until #EOF.
501             elif actual_uri:
502                 output.append(line)
503             else:
504                 error.append(line)
505
506             (line, crash) = self._write_command_and_read_line(input=None)
507
508         return test_output.TestOutput(
509             ''.join(output), self._output_image(), actual_checksum,
510             crash, time.time() - start_time, timeout, ''.join(error))
511
512     def stop(self):
513         if self._proc:
514             self._proc.stdin.close()
515             self._proc.stdout.close()
516             if self._proc.stderr:
517                 self._proc.stderr.close()
518             if sys.platform not in ('win32', 'cygwin'):
519                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
520                 # (see __init__(), above), and anyway we don't want to hang
521                 # the harness if test_shell is buggy, so we wait a couple
522                 # seconds to give test_shell a chance to clean up, but then
523                 # force-kill the process if necessary.
524                 KILL_TIMEOUT = 3.0
525                 timeout = time.time() + KILL_TIMEOUT
526                 # poll() is not threadsafe and can throw OSError due to:
527                 # http://bugs.python.org/issue1731717
528                 while self._proc.poll() is None and time.time() < timeout:
529                     time.sleep(0.1)
530                 # poll() is not threadsafe and can throw OSError due to:
531                 # http://bugs.python.org/issue1731717
532                 if self._proc.poll() is None:
533                     _log.warning('stopping test driver timed out, '
534                                  'killing it')
535                     self._executive.kill_process(self._proc.pid)