f93f9a8d94f0d4d13401495f114f453a47a60f51
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 from __future__ import with_statement
33
34 import codecs
35 import logging
36 import os
37 import re
38 import shutil
39 import signal
40 import subprocess
41 import sys
42 import tempfile
43 import time
44 import webbrowser
45
46 from webkitpy.common.system.executive import Executive
47 from webkitpy.common.system.path import cygpath
48 from webkitpy.layout_tests.layout_package import test_expectations
49
50 import base
51 import http_server
52
53 from webkitpy.common.system.executive import Executive
54 from webkitpy.layout_tests.layout_package import test_expectations
55
56 # Chromium DRT on OSX uses WebKitDriver.
57 if sys.platform == 'darwin':
58     import webkit
59
60 import websocket_server
61
62 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
63
64
65 # FIXME: This function doesn't belong in this package.
66 def check_file_exists(path_to_file, file_description, override_step=None,
67                       logging=True):
68     """Verify the file is present where expected or log an error.
69
70     Args:
71         file_name: The (human friendly) name or description of the file
72             you're looking for (e.g., "HTTP Server"). Used for error logging.
73         override_step: An optional string to be logged if the check fails.
74         logging: Whether or not log the error messages."""
75     if not os.path.exists(path_to_file):
76         if logging:
77             _log.error('Unable to find %s' % file_description)
78             _log.error('    at %s' % path_to_file)
79             if override_step:
80                 _log.error('    %s' % override_step)
81                 _log.error('')
82         return False
83     return True
84
85
86 class ChromiumPort(base.Port):
87     """Abstract base class for Chromium implementations of the Port class."""
88
89     def __init__(self, **kwargs):
90         base.Port.__init__(self, **kwargs)
91         self._chromium_base_dir = None
92
93     def baseline_path(self):
94         return self._webkit_baseline_path(self._name)
95
96     def check_build(self, needs_http):
97         result = True
98
99         dump_render_tree_binary_path = self._path_to_driver()
100         result = check_file_exists(dump_render_tree_binary_path,
101                                     'test driver') and result
102         if result and self.get_option('build'):
103             result = self._check_driver_build_up_to_date(
104                 self.get_option('configuration'))
105         else:
106             _log.error('')
107
108         helper_path = self._path_to_helper()
109         if helper_path:
110             result = check_file_exists(helper_path,
111                                        'layout test helper') and result
112
113         if self.get_option('pixel_tests'):
114             result = self.check_image_diff(
115                 'To override, invoke with --no-pixel-tests') and result
116
117         # It's okay if pretty patch isn't available, but we will at
118         # least log a message.
119         self.check_pretty_patch()
120
121         return result
122
123     def check_sys_deps(self, needs_http):
124         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
125         if self._executive.run_command(cmd, return_exit_code=True):
126             _log.error('System dependencies check failed.')
127             _log.error('To override, invoke with --nocheck-sys-deps')
128             _log.error('')
129             return False
130         return True
131
132     def check_image_diff(self, override_step=None, logging=True):
133         image_diff_path = self._path_to_image_diff()
134         return check_file_exists(image_diff_path, 'image diff exe',
135                                  override_step, logging)
136
137     def diff_image(self, expected_contents, actual_contents,
138                    diff_filename=None):
139         executable = self._path_to_image_diff()
140
141         tempdir = tempfile.mkdtemp()
142         expected_filename = os.path.join(tempdir, "expected.png")
143         with open(expected_filename, 'w+b') as file:
144             file.write(expected_contents)
145         actual_filename = os.path.join(tempdir, "actual.png")
146         with open(actual_filename, 'w+b') as file:
147             file.write(actual_contents)
148
149         if diff_filename:
150             cmd = [executable, '--diff', expected_filename,
151                    actual_filename, diff_filename]
152         else:
153             cmd = [executable, expected_filename, actual_filename]
154
155         result = True
156         try:
157             exit_code = self._executive.run_command(cmd, return_exit_code=True)
158             if exit_code == 0:
159                 # The images are the same.
160                 result = False
161             elif exit_code != 1:
162                 _log.error("image diff returned an exit code of "
163                            + str(exit_code))
164                 # Returning False here causes the script to think that we
165                 # successfully created the diff even though we didn't.  If
166                 # we return True, we think that the images match but the hashes
167                 # don't match.
168                 # FIXME: Figure out why image_diff returns other values.
169                 result = False
170         except OSError, e:
171             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
172                 _compare_available = False
173             else:
174                 raise e
175         finally:
176             shutil.rmtree(tempdir, ignore_errors=True)
177         return result
178
179     def driver_name(self):
180         return "test_shell"
181
182     def path_from_chromium_base(self, *comps):
183         """Returns the full path to path made by joining the top of the
184         Chromium source tree and the list of path components in |*comps|."""
185         if not self._chromium_base_dir:
186             abspath = os.path.abspath(__file__)
187             offset = abspath.find('third_party')
188             if offset == -1:
189                 self._chromium_base_dir = os.path.join(
190                     abspath[0:abspath.find('WebKitTools')],
191                     'WebKit', 'chromium')
192             else:
193                 self._chromium_base_dir = abspath[0:offset]
194         return os.path.join(self._chromium_base_dir, *comps)
195
196     def path_to_test_expectations_file(self):
197         return self.path_from_webkit_base('LayoutTests', 'platform',
198             'chromium', 'test_expectations.txt')
199
200     def results_directory(self):
201         try:
202             return self.path_from_chromium_base('webkit',
203                 self.get_option('configuration'),
204                 self.get_option('results_directory'))
205         except AssertionError:
206             return self._build_path(self.get_option('configuration'),
207                                     self.get_option('results_directory'))
208
209     def setup_test_run(self):
210         # Delete the disk cache if any to ensure a clean test run.
211         dump_render_tree_binary_path = self._path_to_driver()
212         cachedir = os.path.split(dump_render_tree_binary_path)[0]
213         cachedir = os.path.join(cachedir, "cache")
214         if os.path.exists(cachedir):
215             shutil.rmtree(cachedir)
216
217     def create_driver(self, image_path, options):
218         """Starts a new Driver and returns a handle to it."""
219         if options.use_drt and sys.platform == 'darwin':
220             return webkit.WebKitDriver(self, image_path, options,
221                                        executive=self._executive)
222         return ChromiumDriver(self, image_path, options,
223                               executive=self._executive)
224
225     def start_helper(self):
226         helper_path = self._path_to_helper()
227         if helper_path:
228             _log.debug("Starting layout helper %s" % helper_path)
229             # Note: Not thread safe: http://bugs.python.org/issue2320
230             self._helper = subprocess.Popen([helper_path],
231                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
232             is_ready = self._helper.stdout.readline()
233             if not is_ready.startswith('ready'):
234                 _log.error("layout_test_helper failed to be ready")
235
236     def stop_helper(self):
237         if self._helper:
238             _log.debug("Stopping layout test helper")
239             self._helper.stdin.write("x\n")
240             self._helper.stdin.close()
241             # wait() is not threadsafe and can throw OSError due to:
242             # http://bugs.python.org/issue1731717
243             self._helper.wait()
244
245     def test_base_platform_names(self):
246         return ('linux', 'mac', 'win')
247
248     def test_expectations(self):
249         """Returns the test expectations for this port.
250
251         Basically this string should contain the equivalent of a
252         test_expectations file. See test_expectations.py for more details."""
253         expectations_path = self.path_to_test_expectations_file()
254         with codecs.open(expectations_path, "r", "utf-8") as file:
255             return file.read()
256
257     def test_expectations_overrides(self):
258         # FIXME: This drt_overrides handling should be removed when we switch
259         # from tes_shell to DRT.
260         drt_overrides = ''
261         if self.get_option('use_drt'):
262             drt_overrides_path = self.path_from_webkit_base('LayoutTests',
263                 'platform', 'chromium', 'drt_expectations.txt')
264             if os.path.exists(drt_overrides_path):
265                 with codecs.open(drt_overrides_path, "r", "utf-8") as file:
266                     drt_overrides = file.read()
267
268         try:
269             overrides_path = self.path_from_chromium_base('webkit', 'tools',
270                 'layout_tests', 'test_expectations.txt')
271         except AssertionError:
272             return None
273         if not os.path.exists(overrides_path):
274             return None
275         with codecs.open(overrides_path, "r", "utf-8") as file:
276             return file.read() + drt_overrides
277
278     def skipped_layout_tests(self, extra_test_files=None):
279         expectations_str = self.test_expectations()
280         overrides_str = self.test_expectations_overrides()
281         test_platform_name = self.test_platform_name()
282         is_debug_mode = False
283
284         all_test_files = self.tests([])
285         if extra_test_files:
286             all_test_files.update(extra_test_files)
287
288         expectations = test_expectations.TestExpectations(
289             self, all_test_files, expectations_str, test_platform_name,
290             is_debug_mode, is_lint_mode=True, overrides=overrides_str)
291         tests_dir = self.layout_tests_dir()
292         return [self.relative_test_filename(test)
293                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
294
295     def test_platform_names(self):
296         return self.test_base_platform_names() + ('win-xp',
297             'win-vista', 'win-7')
298
299     def test_platform_name_to_name(self, test_platform_name):
300         if test_platform_name in self.test_platform_names():
301             return 'chromium-' + test_platform_name
302         raise ValueError('Unsupported test_platform_name: %s' %
303                          test_platform_name)
304
305     def test_repository_paths(self):
306         # Note: for JSON file's backward-compatibility we use 'chrome' rather
307         # than 'chromium' here.
308         repos = super(ChromiumPort, self).test_repository_paths()
309         repos.append(('chrome', self.path_from_chromium_base()))
310         return repos
311
312     #
313     # PROTECTED METHODS
314     #
315     # These routines should only be called by other methods in this file
316     # or any subclasses.
317     #
318
319     def _check_driver_build_up_to_date(self, configuration):
320         if configuration in ('Debug', 'Release'):
321             try:
322                 debug_path = self._path_to_driver('Debug')
323                 release_path = self._path_to_driver('Release')
324
325                 debug_mtime = os.stat(debug_path).st_mtime
326                 release_mtime = os.stat(release_path).st_mtime
327
328                 if (debug_mtime > release_mtime and configuration == 'Release' or
329                     release_mtime > debug_mtime and configuration == 'Debug'):
330                     _log.warning('You are not running the most '
331                                  'recent DumpRenderTree binary. You need to '
332                                  'pass --debug or not to select between '
333                                  'Debug and Release.')
334                     _log.warning('')
335             # This will fail if we don't have both a debug and release binary.
336             # That's fine because, in this case, we must already be running the
337             # most up-to-date one.
338             except OSError:
339                 pass
340         return True
341
342     def _chromium_baseline_path(self, platform):
343         if platform is None:
344             platform = self.name()
345         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
346
347     def _convert_path(self, path):
348         """Handles filename conversion for subprocess command line args."""
349         # See note above in diff_image() for why we need this.
350         if sys.platform == 'cygwin':
351             return cygpath(path)
352         return path
353
354     def _path_to_image_diff(self):
355         binary_name = 'image_diff'
356         if self.get_option('use_drt'):
357             binary_name = 'ImageDiff'
358         return self._build_path(self.get_option('configuration'), binary_name)
359
360
361 class ChromiumDriver(base.Driver):
362     """Abstract interface for test_shell."""
363
364     def __init__(self, port, image_path, options, executive=Executive()):
365         self._port = port
366         self._options = options
367         self._image_path = image_path
368         self._executive = executive
369
370     def _driver_args(self):
371         driver_args = []
372         if self._image_path:
373             # See note above in diff_image() for why we need _convert_path().
374             driver_args.append("--pixel-tests=" +
375                                self._port._convert_path(self._image_path))
376
377         if self._port.get_option('use_drt'):
378             driver_args.append('--test-shell')
379         else:
380             driver_args.append('--layout-tests')
381
382         if self._port.get_option('startup_dialog'):
383             driver_args.append('--testshell-startup-dialog')
384
385         if self._port.get_option('gp_fault_error_box'):
386             driver_args.append('--gp-fault-error-box')
387
388         if self._options.js_flags is not None:
389             driver_args.append('--js-flags="' + self._options.js_flags + '"')
390
391         if self._options.multiple_loads is not None and self._options.multiple_loads > 0:
392             driver_args.append('--multiple-loads=' + str(self._options.multiple_loads))
393
394         if self._port.get_option('accelerated_compositing'):
395             driver_args.append('--enable-accelerated-compositing')
396
397         if self._port.get_option('accelerated_2d_canvas'):
398             driver_args.append('--enable-accelerated-2d-canvas')
399         return driver_args
400
401     def start(self):
402         # FIXME: Should be an error to call this method twice.
403         cmd = self._command_wrapper(self._port.get_option('wrapper'))
404         cmd.append(self._port._path_to_driver())
405         cmd += self._driver_args()
406
407         # We need to pass close_fds=True to work around Python bug #2320
408         # (otherwise we can hang when we kill DumpRenderTree when we are running
409         # multiple threads). See http://bugs.python.org/issue2320 .
410         # Note that close_fds isn't supported on Windows, but this bug only
411         # shows up on Mac and Linux.
412         close_flag = sys.platform not in ('win32', 'cygwin')
413         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
414                                       stdout=subprocess.PIPE,
415                                       stderr=subprocess.STDOUT,
416                                       close_fds=close_flag)
417
418     def poll(self):
419         # poll() is not threadsafe and can throw OSError due to:
420         # http://bugs.python.org/issue1731717
421         return self._proc.poll()
422
423     def _write_command_and_read_line(self, input=None):
424         """Returns a tuple: (line, did_crash)"""
425         try:
426             if input:
427                 if isinstance(input, unicode):
428                     # TestShell expects utf-8
429                     input = input.encode("utf-8")
430                 self._proc.stdin.write(input)
431             # DumpRenderTree text output is always UTF-8.  However some tests
432             # (e.g. webarchive) may spit out binary data instead of text so we
433             # don't bother to decode the output (for either DRT or test_shell).
434             line = self._proc.stdout.readline()
435             # We could assert() here that line correctly decodes as UTF-8.
436             return (line, False)
437         except IOError, e:
438             _log.error("IOError communicating w/ test_shell: " + str(e))
439             return (None, True)
440
441     def _test_shell_command(self, uri, timeoutms, checksum):
442         cmd = uri
443         if timeoutms:
444             cmd += ' ' + str(timeoutms)
445         if checksum:
446             cmd += ' ' + checksum
447         cmd += "\n"
448         return cmd
449
450     def run_test(self, uri, timeoutms, checksum):
451         output = []
452         error = []
453         crash = False
454         timeout = False
455         actual_uri = None
456         actual_checksum = None
457
458         start_time = time.time()
459
460         cmd = self._test_shell_command(uri, timeoutms, checksum)
461         (line, crash) = self._write_command_and_read_line(input=cmd)
462
463         while not crash and line.rstrip() != "#EOF":
464             # Make sure we haven't crashed.
465             if line == '' and self.poll() is not None:
466                 # This is hex code 0xc000001d, which is used for abrupt
467                 # termination. This happens if we hit ctrl+c from the prompt
468                 # and we happen to be waiting on test_shell.
469                 # sdoyon: Not sure for which OS and in what circumstances the
470                 # above code is valid. What works for me under Linux to detect
471                 # ctrl+c is for the subprocess returncode to be negative
472                 # SIGINT. And that agrees with the subprocess documentation.
473                 if (-1073741510 == self._proc.returncode or
474                     - signal.SIGINT == self._proc.returncode):
475                     raise KeyboardInterrupt
476                 crash = True
477                 break
478
479             # Don't include #URL lines in our output
480             if line.startswith("#URL:"):
481                 actual_uri = line.rstrip()[5:]
482                 if uri != actual_uri:
483                     # GURL capitalizes the drive letter of a file URL.
484                     if (not re.search("^file:///[a-z]:", uri) or
485                         uri.lower() != actual_uri.lower()):
486                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
487                                    (uri, actual_uri))
488                         raise AssertionError("test out of sync")
489             elif line.startswith("#MD5:"):
490                 actual_checksum = line.rstrip()[5:]
491             elif line.startswith("#TEST_TIMED_OUT"):
492                 timeout = True
493                 # Test timed out, but we still need to read until #EOF.
494             elif actual_uri:
495                 output.append(line)
496             else:
497                 error.append(line)
498
499             (line, crash) = self._write_command_and_read_line(input=None)
500
501         return (crash, timeout, actual_checksum, ''.join(output),
502                 ''.join(error))
503
504     def stop(self):
505         if self._proc:
506             self._proc.stdin.close()
507             self._proc.stdout.close()
508             if self._proc.stderr:
509                 self._proc.stderr.close()
510             if sys.platform not in ('win32', 'cygwin'):
511                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
512                 # (see __init__(), above), and anyway we don't want to hang
513                 # the harness if test_shell is buggy, so we wait a couple
514                 # seconds to give test_shell a chance to clean up, but then
515                 # force-kill the process if necessary.
516                 KILL_TIMEOUT = 3.0
517                 timeout = time.time() + KILL_TIMEOUT
518                 # poll() is not threadsafe and can throw OSError due to:
519                 # http://bugs.python.org/issue1731717
520                 while self._proc.poll() is None and time.time() < timeout:
521                     time.sleep(0.1)
522                 # poll() is not threadsafe and can throw OSError due to:
523                 # http://bugs.python.org/issue1731717
524                 if self._proc.poll() is None:
525                     _log.warning('stopping test driver timed out, '
526                                  'killing it')
527                     self._executive.kill_process(self._proc.pid)