2011-02-03 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 import errno
33 import logging
34 import re
35 import signal
36 import subprocess
37 import sys
38 import time
39 import webbrowser
40
41 from webkitpy.common.system import executive
42 from webkitpy.common.system.path import cygpath
43 from webkitpy.layout_tests.layout_package import test_expectations
44
45 import base
46 import http_server
47 import websocket_server
48
49 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
50
51
52 # FIXME: This function doesn't belong in this package.
53 class ChromiumPort(base.Port):
54     """Abstract base class for Chromium implementations of the Port class."""
55
56     def __init__(self, **kwargs):
57         base.Port.__init__(self, **kwargs)
58         self._chromium_base_dir = None
59
60     def _check_file_exists(self, path_to_file, file_description,
61                            override_step=None, logging=True):
62         """Verify the file is present where expected or log an error.
63
64         Args:
65             file_name: The (human friendly) name or description of the file
66                 you're looking for (e.g., "HTTP Server"). Used for error logging.
67             override_step: An optional string to be logged if the check fails.
68             logging: Whether or not log the error messages."""
69         if not self._filesystem.exists(path_to_file):
70             if logging:
71                 _log.error('Unable to find %s' % file_description)
72                 _log.error('    at %s' % path_to_file)
73                 if override_step:
74                     _log.error('    %s' % override_step)
75                     _log.error('')
76             return False
77         return True
78
79
80     def baseline_path(self):
81         return self._webkit_baseline_path(self._name)
82
83     def check_build(self, needs_http):
84         result = True
85
86         dump_render_tree_binary_path = self._path_to_driver()
87         result = self._check_file_exists(dump_render_tree_binary_path,
88                                          'test driver') and result
89         if result and self.get_option('build'):
90             result = self._check_driver_build_up_to_date(
91                 self.get_option('configuration'))
92         else:
93             _log.error('')
94
95         helper_path = self._path_to_helper()
96         if helper_path:
97             result = self._check_file_exists(helper_path,
98                                              'layout test helper') and result
99
100         if self.get_option('pixel_tests'):
101             result = self.check_image_diff(
102                 'To override, invoke with --no-pixel-tests') and result
103
104         # It's okay if pretty patch isn't available, but we will at
105         # least log a message.
106         self.check_pretty_patch()
107
108         return result
109
110     def check_sys_deps(self, needs_http):
111         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
112
113         local_error = executive.ScriptError()
114
115         def error_handler(script_error):
116             local_error.exit_code = script_error.exit_code
117
118         output = self._executive.run_command(cmd, error_handler=error_handler)
119         if local_error.exit_code:
120             _log.error('System dependencies check failed.')
121             _log.error('To override, invoke with --nocheck-sys-deps')
122             _log.error('')
123             _log.error(output)
124             return False
125         return True
126
127     def check_image_diff(self, override_step=None, logging=True):
128         image_diff_path = self._path_to_image_diff()
129         return self._check_file_exists(image_diff_path, 'image diff exe',
130                                        override_step, logging)
131
132     def diff_image(self, expected_contents, actual_contents,
133                    diff_filename=None):
134         executable = self._path_to_image_diff()
135
136         tempdir = self._filesystem.mkdtemp()
137         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
138         self._filesystem.write_binary_file(expected_filename, expected_contents)
139         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
140         self._filesystem.write_binary_file(actual_filename, actual_contents)
141
142         if diff_filename:
143             cmd = [executable, '--diff', expected_filename,
144                    actual_filename, diff_filename]
145         else:
146             cmd = [executable, expected_filename, actual_filename]
147
148         result = True
149         try:
150             exit_code = self._executive.run_command(cmd, return_exit_code=True)
151             if exit_code == 0:
152                 # The images are the same.
153                 result = False
154             elif exit_code != 1:
155                 _log.error("image diff returned an exit code of "
156                            + str(exit_code))
157                 # Returning False here causes the script to think that we
158                 # successfully created the diff even though we didn't.  If
159                 # we return True, we think that the images match but the hashes
160                 # don't match.
161                 # FIXME: Figure out why image_diff returns other values.
162                 result = False
163         except OSError, e:
164             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
165                 _compare_available = False
166             else:
167                 raise e
168         finally:
169             self._filesystem.rmtree(str(tempdir))
170         return result
171
172     def driver_name(self):
173         return "DumpRenderTree"
174
175     def path_from_chromium_base(self, *comps):
176         """Returns the full path to path made by joining the top of the
177         Chromium source tree and the list of path components in |*comps|."""
178         if not self._chromium_base_dir:
179             abspath = self._filesystem.abspath(__file__)
180             offset = abspath.find('third_party')
181             if offset == -1:
182                 self._chromium_base_dir = self._filesystem.join(
183                     abspath[0:abspath.find('Tools')],
184                     'Source', 'WebKit', 'chromium')
185             else:
186                 self._chromium_base_dir = abspath[0:offset]
187         return self._filesystem.join(self._chromium_base_dir, *comps)
188
189     def path_to_test_expectations_file(self):
190         return self.path_from_webkit_base('LayoutTests', 'platform',
191             'chromium', 'test_expectations.txt')
192
193     def results_directory(self):
194         try:
195             return self.path_from_chromium_base('webkit',
196                 self.get_option('configuration'),
197                 self.get_option('results_directory'))
198         except AssertionError:
199             return self._build_path(self.get_option('configuration'),
200                                     self.get_option('results_directory'))
201
202     def setup_test_run(self):
203         # Delete the disk cache if any to ensure a clean test run.
204         dump_render_tree_binary_path = self._path_to_driver()
205         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
206         cachedir = self._filesystem.join(cachedir, "cache")
207         if self._filesystem.exists(cachedir):
208             self._filesystem.rmtree(cachedir)
209
210     def create_driver(self, worker_number):
211         """Starts a new Driver and returns a handle to it."""
212         return ChromiumDriver(self, worker_number)
213
214     def start_helper(self):
215         helper_path = self._path_to_helper()
216         if helper_path:
217             _log.debug("Starting layout helper %s" % helper_path)
218             # Note: Not thread safe: http://bugs.python.org/issue2320
219             self._helper = subprocess.Popen([helper_path],
220                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
221             is_ready = self._helper.stdout.readline()
222             if not is_ready.startswith('ready'):
223                 _log.error("layout_test_helper failed to be ready")
224
225     def stop_helper(self):
226         if self._helper:
227             _log.debug("Stopping layout test helper")
228             self._helper.stdin.write("x\n")
229             self._helper.stdin.close()
230             # wait() is not threadsafe and can throw OSError due to:
231             # http://bugs.python.org/issue1731717
232             self._helper.wait()
233
234     def test_expectations(self):
235         """Returns the test expectations for this port.
236
237         Basically this string should contain the equivalent of a
238         test_expectations file. See test_expectations.py for more details."""
239         expectations_path = self.path_to_test_expectations_file()
240         return self._filesystem.read_text_file(expectations_path)
241
242     def test_expectations_overrides(self):
243         try:
244             overrides_path = self.path_from_chromium_base('webkit', 'tools',
245                 'layout_tests', 'test_expectations.txt')
246         except AssertionError:
247             return None
248         if not self._filesystem.exists(overrides_path):
249             return None
250         return self._filesystem.read_text_file(overrides_path)
251
252     def skipped_layout_tests(self, extra_test_files=None):
253         expectations_str = self.test_expectations()
254         overrides_str = self.test_expectations_overrides()
255         test_platform_name = self.test_platform_name()
256         is_debug_mode = False
257
258         all_test_files = self.tests([])
259         if extra_test_files:
260             all_test_files.update(extra_test_files)
261
262         expectations = test_expectations.TestExpectations(
263             self, all_test_files, expectations_str, self.test_configuration(),
264             is_lint_mode=False, overrides=overrides_str)
265         tests_dir = self.layout_tests_dir()
266         return [self.relative_test_filename(test)
267                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
268
269     def test_platform_names(self):
270         return ('mac', 'win', 'linux', 'win-xp', 'win-vista', 'win-7')
271
272     def test_platform_name_to_name(self, test_platform_name):
273         if test_platform_name in self.test_platform_names():
274             return 'chromium-' + test_platform_name
275         raise ValueError('Unsupported test_platform_name: %s' %
276                          test_platform_name)
277
278     def test_repository_paths(self):
279         # Note: for JSON file's backward-compatibility we use 'chrome' rather
280         # than 'chromium' here.
281         repos = super(ChromiumPort, self).test_repository_paths()
282         repos.append(('chrome', self.path_from_chromium_base()))
283         return repos
284
285     #
286     # PROTECTED METHODS
287     #
288     # These routines should only be called by other methods in this file
289     # or any subclasses.
290     #
291
292     def _check_driver_build_up_to_date(self, configuration):
293         if configuration in ('Debug', 'Release'):
294             try:
295                 debug_path = self._path_to_driver('Debug')
296                 release_path = self._path_to_driver('Release')
297
298                 debug_mtime = self._filesystem.mtime(debug_path)
299                 release_mtime = self._filesystem.mtime(release_path)
300
301                 if (debug_mtime > release_mtime and configuration == 'Release' or
302                     release_mtime > debug_mtime and configuration == 'Debug'):
303                     _log.warning('You are not running the most '
304                                  'recent DumpRenderTree binary. You need to '
305                                  'pass --debug or not to select between '
306                                  'Debug and Release.')
307                     _log.warning('')
308             # This will fail if we don't have both a debug and release binary.
309             # That's fine because, in this case, we must already be running the
310             # most up-to-date one.
311             except OSError:
312                 pass
313         return True
314
315     def _chromium_baseline_path(self, platform):
316         if platform is None:
317             platform = self.name()
318         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
319
320     def _convert_path(self, path):
321         """Handles filename conversion for subprocess command line args."""
322         # See note above in diff_image() for why we need this.
323         if sys.platform == 'cygwin':
324             return cygpath(path)
325         return path
326
327     def _path_to_image_diff(self):
328         binary_name = 'ImageDiff'
329         return self._build_path(self.get_option('configuration'), binary_name)
330
331
332 class ChromiumDriver(base.Driver):
333     """Abstract interface for DRT."""
334
335     def __init__(self, port, worker_number):
336         self._port = port
337         self._worker_number = worker_number
338         self._image_path = None
339         if self._port.get_option('pixel_tests'):
340             self._image_path = self._port._filesystem.join(
341                 self._port.get_option('results_directory'),
342                 'png_result%s.png' % self._worker_number)
343
344     def cmd_line(self):
345         cmd = self._command_wrapper(self._port.get_option('wrapper'))
346         cmd.append(self._port._path_to_driver())
347         if self._port.get_option('pixel_tests'):
348             # See note above in diff_image() for why we need _convert_path().
349             cmd.append("--pixel-tests=" +
350                        self._port._convert_path(self._image_path))
351
352         cmd.append('--test-shell')
353
354         if self._port.get_option('startup_dialog'):
355             cmd.append('--testshell-startup-dialog')
356
357         if self._port.get_option('gp_fault_error_box'):
358             cmd.append('--gp-fault-error-box')
359
360         if self._port.get_option('js_flags') is not None:
361             cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"')
362
363         if self._port.get_option('stress_opt'):
364             cmd.append('--stress-opt')
365
366         if self._port.get_option('stress_deopt'):
367             cmd.append('--stress-deopt')
368
369         if self._port.get_option('accelerated_compositing'):
370             cmd.append('--enable-accelerated-compositing')
371         if self._port.get_option('accelerated_2d_canvas'):
372             cmd.append('--enable-accelerated-2d-canvas')
373         if self._port.get_option('enable_hardware_gpu'):
374             cmd.append('--enable-hardware-gpu')
375         return cmd
376
377     def start(self):
378         # FIXME: Should be an error to call this method twice.
379         cmd = self.cmd_line()
380
381         # We need to pass close_fds=True to work around Python bug #2320
382         # (otherwise we can hang when we kill DumpRenderTree when we are running
383         # multiple threads). See http://bugs.python.org/issue2320 .
384         # Note that close_fds isn't supported on Windows, but this bug only
385         # shows up on Mac and Linux.
386         close_flag = sys.platform not in ('win32', 'cygwin')
387         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
388                                       stdout=subprocess.PIPE,
389                                       stderr=subprocess.STDOUT,
390                                       close_fds=close_flag)
391
392     def poll(self):
393         # poll() is not threadsafe and can throw OSError due to:
394         # http://bugs.python.org/issue1731717
395         return self._proc.poll()
396
397     def _write_command_and_read_line(self, input=None):
398         """Returns a tuple: (line, did_crash)"""
399         try:
400             if input:
401                 if isinstance(input, unicode):
402                     # DRT expects utf-8
403                     input = input.encode("utf-8")
404                 self._proc.stdin.write(input)
405             # DumpRenderTree text output is always UTF-8.  However some tests
406             # (e.g. webarchive) may spit out binary data instead of text so we
407             # don't bother to decode the output.
408             line = self._proc.stdout.readline()
409             # We could assert() here that line correctly decodes as UTF-8.
410             return (line, False)
411         except IOError, e:
412             _log.error("IOError communicating w/ DRT: " + str(e))
413             return (None, True)
414
415     def _test_shell_command(self, uri, timeoutms, checksum):
416         cmd = uri
417         if timeoutms:
418             cmd += ' ' + str(timeoutms)
419         if checksum:
420             cmd += ' ' + checksum
421         cmd += "\n"
422         return cmd
423
424     def _output_image(self):
425         """Returns the image output which driver generated."""
426         png_path = self._image_path
427         if png_path and self._port._filesystem.isfile(png_path):
428             return self._port._filesystem.read_binary_file(png_path)
429         else:
430             return None
431
432     def _output_image_with_retry(self):
433         # Retry a few more times because open() sometimes fails on Windows,
434         # raising "IOError: [Errno 13] Permission denied:"
435         retry_num = 50
436         timeout_seconds = 5.0
437         for i in range(retry_num):
438             try:
439                 return self._output_image()
440             except IOError, e:
441                 if e.errno == errno.EACCES:
442                     time.sleep(timeout_seconds / retry_num)
443                 else:
444                     raise e
445         return self._output_image()
446
447     def run_test(self, driver_input):
448         output = []
449         error = []
450         crash = False
451         timeout = False
452         actual_uri = None
453         actual_checksum = None
454
455         start_time = time.time()
456
457         uri = self._port.filename_to_uri(driver_input.filename)
458         cmd = self._test_shell_command(uri, driver_input.timeout,
459                                        driver_input.image_hash)
460         (line, crash) = self._write_command_and_read_line(input=cmd)
461
462         while not crash and line.rstrip() != "#EOF":
463             # Make sure we haven't crashed.
464             if line == '' and self.poll() is not None:
465                 # This is hex code 0xc000001d, which is used for abrupt
466                 # termination. This happens if we hit ctrl+c from the prompt
467                 # and we happen to be waiting on DRT.
468                 # sdoyon: Not sure for which OS and in what circumstances the
469                 # above code is valid. What works for me under Linux to detect
470                 # ctrl+c is for the subprocess returncode to be negative
471                 # SIGINT. And that agrees with the subprocess documentation.
472                 if (-1073741510 == self._proc.returncode or
473                     - signal.SIGINT == self._proc.returncode):
474                     raise KeyboardInterrupt
475                 crash = True
476                 break
477
478             # Don't include #URL lines in our output
479             if line.startswith("#URL:"):
480                 actual_uri = line.rstrip()[5:]
481                 if uri != actual_uri:
482                     # GURL capitalizes the drive letter of a file URL.
483                     if (not re.search("^file:///[a-z]:", uri) or
484                         uri.lower() != actual_uri.lower()):
485                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
486                                    (uri, actual_uri))
487                         raise AssertionError("test out of sync")
488             elif line.startswith("#MD5:"):
489                 actual_checksum = line.rstrip()[5:]
490             elif line.startswith("#TEST_TIMED_OUT"):
491                 timeout = True
492                 # Test timed out, but we still need to read until #EOF.
493             elif actual_uri:
494                 output.append(line)
495             else:
496                 error.append(line)
497
498             (line, crash) = self._write_command_and_read_line(input=None)
499
500         run_time = time.time() - start_time
501         return base.DriverOutput(
502             ''.join(output), self._output_image_with_retry(), actual_checksum,
503             crash, run_time, timeout, ''.join(error))
504
505     def stop(self):
506         if self._proc:
507             self._proc.stdin.close()
508             self._proc.stdout.close()
509             if self._proc.stderr:
510                 self._proc.stderr.close()
511             if sys.platform not in ('win32', 'cygwin'):
512                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
513                 # (see __init__(), above), and anyway we don't want to hang
514                 # the harness if DRT is buggy, so we wait a couple
515                 # seconds to give DRT a chance to clean up, but then
516                 # force-kill the process if necessary.
517                 KILL_TIMEOUT = 3.0
518                 timeout = time.time() + KILL_TIMEOUT
519                 # poll() is not threadsafe and can throw OSError due to:
520                 # http://bugs.python.org/issue1731717
521                 while self._proc.poll() is None and time.time() < timeout:
522                     time.sleep(0.1)
523                 # poll() is not threadsafe and can throw OSError due to:
524                 # http://bugs.python.org/issue1731717
525                 if self._proc.poll() is None:
526                     _log.warning('stopping test driver timed out, '
527                                  'killing it')
528                     self._port._executive.kill_process(self._proc.pid)