e15a035cdf4834bb800ae47469b1e3169f1afec1
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / chromium.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Chromium implementations of the Port interface."""
31
32 import errno
33 import logging
34 import re
35 import signal
36 import subprocess
37 import sys
38 import time
39 import webbrowser
40
41 from webkitpy.common.system import executive
42 from webkitpy.common.system.path import cygpath
43 from webkitpy.layout_tests.layout_package import test_expectations
44 from webkitpy.layout_tests.layout_package import test_output
45
46 import base
47 import http_server
48 import websocket_server
49
50 _log = logging.getLogger("webkitpy.layout_tests.port.chromium")
51
52
53 # FIXME: This function doesn't belong in this package.
54 class ChromiumPort(base.Port):
55     """Abstract base class for Chromium implementations of the Port class."""
56
57     def __init__(self, **kwargs):
58         base.Port.__init__(self, **kwargs)
59         self._chromium_base_dir = None
60
61     def _check_file_exists(self, path_to_file, file_description,
62                            override_step=None, logging=True):
63         """Verify the file is present where expected or log an error.
64
65         Args:
66             file_name: The (human friendly) name or description of the file
67                 you're looking for (e.g., "HTTP Server"). Used for error logging.
68             override_step: An optional string to be logged if the check fails.
69             logging: Whether or not log the error messages."""
70         if not self._filesystem.exists(path_to_file):
71             if logging:
72                 _log.error('Unable to find %s' % file_description)
73                 _log.error('    at %s' % path_to_file)
74                 if override_step:
75                     _log.error('    %s' % override_step)
76                     _log.error('')
77             return False
78         return True
79
80
81     def baseline_path(self):
82         return self._webkit_baseline_path(self._name)
83
84     def check_build(self, needs_http):
85         result = True
86
87         dump_render_tree_binary_path = self._path_to_driver()
88         result = self._check_file_exists(dump_render_tree_binary_path,
89                                          'test driver') and result
90         if result and self.get_option('build'):
91             result = self._check_driver_build_up_to_date(
92                 self.get_option('configuration'))
93         else:
94             _log.error('')
95
96         helper_path = self._path_to_helper()
97         if helper_path:
98             result = self._check_file_exists(helper_path,
99                                              'layout test helper') and result
100
101         if self.get_option('pixel_tests'):
102             result = self.check_image_diff(
103                 'To override, invoke with --no-pixel-tests') and result
104
105         # It's okay if pretty patch isn't available, but we will at
106         # least log a message.
107         self.check_pretty_patch()
108
109         return result
110
111     def check_sys_deps(self, needs_http):
112         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
113
114         local_error = executive.ScriptError()
115
116         def error_handler(script_error):
117             local_error.exit_code = script_error.exit_code
118
119         output = self._executive.run_command(cmd, error_handler=error_handler)
120         if local_error.exit_code:
121             _log.error('System dependencies check failed.')
122             _log.error('To override, invoke with --nocheck-sys-deps')
123             _log.error('')
124             _log.error(output)
125             return False
126         return True
127
128     def check_image_diff(self, override_step=None, logging=True):
129         image_diff_path = self._path_to_image_diff()
130         return self._check_file_exists(image_diff_path, 'image diff exe',
131                                        override_step, logging)
132
133     def diff_image(self, expected_contents, actual_contents,
134                    diff_filename=None):
135         executable = self._path_to_image_diff()
136
137         tempdir = self._filesystem.mkdtemp()
138         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
139         self._filesystem.write_binary_file(expected_filename, expected_contents)
140         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
141         self._filesystem.write_binary_file(actual_filename, actual_contents)
142
143         if diff_filename:
144             cmd = [executable, '--diff', expected_filename,
145                    actual_filename, diff_filename]
146         else:
147             cmd = [executable, expected_filename, actual_filename]
148
149         result = True
150         try:
151             exit_code = self._executive.run_command(cmd, return_exit_code=True)
152             if exit_code == 0:
153                 # The images are the same.
154                 result = False
155             elif exit_code != 1:
156                 _log.error("image diff returned an exit code of "
157                            + str(exit_code))
158                 # Returning False here causes the script to think that we
159                 # successfully created the diff even though we didn't.  If
160                 # we return True, we think that the images match but the hashes
161                 # don't match.
162                 # FIXME: Figure out why image_diff returns other values.
163                 result = False
164         except OSError, e:
165             if e.errno == errno.ENOENT or e.errno == errno.EACCES:
166                 _compare_available = False
167             else:
168                 raise e
169         finally:
170             self._filesystem.rmtree(str(tempdir))
171         return result
172
173     def driver_name(self):
174         return "DumpRenderTree"
175
176     def path_from_chromium_base(self, *comps):
177         """Returns the full path to path made by joining the top of the
178         Chromium source tree and the list of path components in |*comps|."""
179         if not self._chromium_base_dir:
180             abspath = self._filesystem.abspath(__file__)
181             offset = abspath.find('third_party')
182             if offset == -1:
183                 self._chromium_base_dir = self._filesystem.join(
184                     abspath[0:abspath.find('Tools')],
185                     'Source', 'WebKit', 'chromium')
186             else:
187                 self._chromium_base_dir = abspath[0:offset]
188         return self._filesystem.join(self._chromium_base_dir, *comps)
189
190     def path_to_test_expectations_file(self):
191         return self.path_from_webkit_base('LayoutTests', 'platform',
192             'chromium', 'test_expectations.txt')
193
194     def results_directory(self):
195         try:
196             return self.path_from_chromium_base('webkit',
197                 self.get_option('configuration'),
198                 self.get_option('results_directory'))
199         except AssertionError:
200             return self._build_path(self.get_option('configuration'),
201                                     self.get_option('results_directory'))
202
203     def setup_test_run(self):
204         # Delete the disk cache if any to ensure a clean test run.
205         dump_render_tree_binary_path = self._path_to_driver()
206         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
207         cachedir = self._filesystem.join(cachedir, "cache")
208         if self._filesystem.exists(cachedir):
209             self._filesystem.rmtree(cachedir)
210
211     def create_driver(self, worker_number):
212         """Starts a new Driver and returns a handle to it."""
213         return ChromiumDriver(self, worker_number)
214
215     def start_helper(self):
216         helper_path = self._path_to_helper()
217         if helper_path:
218             _log.debug("Starting layout helper %s" % helper_path)
219             # Note: Not thread safe: http://bugs.python.org/issue2320
220             self._helper = subprocess.Popen([helper_path],
221                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
222             is_ready = self._helper.stdout.readline()
223             if not is_ready.startswith('ready'):
224                 _log.error("layout_test_helper failed to be ready")
225
226     def stop_helper(self):
227         if self._helper:
228             _log.debug("Stopping layout test helper")
229             self._helper.stdin.write("x\n")
230             self._helper.stdin.close()
231             # wait() is not threadsafe and can throw OSError due to:
232             # http://bugs.python.org/issue1731717
233             self._helper.wait()
234
235     def test_expectations(self):
236         """Returns the test expectations for this port.
237
238         Basically this string should contain the equivalent of a
239         test_expectations file. See test_expectations.py for more details."""
240         expectations_path = self.path_to_test_expectations_file()
241         return self._filesystem.read_text_file(expectations_path)
242
243     def test_expectations_overrides(self):
244         try:
245             overrides_path = self.path_from_chromium_base('webkit', 'tools',
246                 'layout_tests', 'test_expectations.txt')
247         except AssertionError:
248             return None
249         if not self._filesystem.exists(overrides_path):
250             return None
251         return self._filesystem.read_text_file(overrides_path)
252
253     def skipped_layout_tests(self, extra_test_files=None):
254         expectations_str = self.test_expectations()
255         overrides_str = self.test_expectations_overrides()
256         test_platform_name = self.test_platform_name()
257         is_debug_mode = False
258
259         all_test_files = self.tests([])
260         if extra_test_files:
261             all_test_files.update(extra_test_files)
262
263         expectations = test_expectations.TestExpectations(
264             self, all_test_files, expectations_str, self.test_configuration(),
265             is_lint_mode=False, overrides=overrides_str)
266         tests_dir = self.layout_tests_dir()
267         return [self.relative_test_filename(test)
268                 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
269
270     def test_platform_names(self):
271         return ('mac', 'win', 'linux', 'win-xp', 'win-vista', 'win-7')
272
273     def test_platform_name_to_name(self, test_platform_name):
274         if test_platform_name in self.test_platform_names():
275             return 'chromium-' + test_platform_name
276         raise ValueError('Unsupported test_platform_name: %s' %
277                          test_platform_name)
278
279     def test_repository_paths(self):
280         # Note: for JSON file's backward-compatibility we use 'chrome' rather
281         # than 'chromium' here.
282         repos = super(ChromiumPort, self).test_repository_paths()
283         repos.append(('chrome', self.path_from_chromium_base()))
284         return repos
285
286     #
287     # PROTECTED METHODS
288     #
289     # These routines should only be called by other methods in this file
290     # or any subclasses.
291     #
292
293     def _check_driver_build_up_to_date(self, configuration):
294         if configuration in ('Debug', 'Release'):
295             try:
296                 debug_path = self._path_to_driver('Debug')
297                 release_path = self._path_to_driver('Release')
298
299                 debug_mtime = self._filesystem.mtime(debug_path)
300                 release_mtime = self._filesystem.mtime(release_path)
301
302                 if (debug_mtime > release_mtime and configuration == 'Release' or
303                     release_mtime > debug_mtime and configuration == 'Debug'):
304                     _log.warning('You are not running the most '
305                                  'recent DumpRenderTree binary. You need to '
306                                  'pass --debug or not to select between '
307                                  'Debug and Release.')
308                     _log.warning('')
309             # This will fail if we don't have both a debug and release binary.
310             # That's fine because, in this case, we must already be running the
311             # most up-to-date one.
312             except OSError:
313                 pass
314         return True
315
316     def _chromium_baseline_path(self, platform):
317         if platform is None:
318             platform = self.name()
319         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
320
321     def _convert_path(self, path):
322         """Handles filename conversion for subprocess command line args."""
323         # See note above in diff_image() for why we need this.
324         if sys.platform == 'cygwin':
325             return cygpath(path)
326         return path
327
328     def _path_to_image_diff(self):
329         binary_name = 'ImageDiff'
330         return self._build_path(self.get_option('configuration'), binary_name)
331
332
333 class ChromiumDriver(base.Driver):
334     """Abstract interface for DRT."""
335
336     def __init__(self, port, worker_number):
337         self._port = port
338         self._worker_number = worker_number
339         self._image_path = None
340         if self._port.get_option('pixel_tests'):
341             self._image_path = self._port._filesystem.join(
342                 self._port.get_option('results_directory'),
343                 'png_result%s.png' % self._worker_number)
344
345     def cmd_line(self):
346         cmd = self._command_wrapper(self._port.get_option('wrapper'))
347         cmd.append(self._port._path_to_driver())
348         if self._port.get_option('pixel_tests'):
349             # See note above in diff_image() for why we need _convert_path().
350             cmd.append("--pixel-tests=" +
351                        self._port._convert_path(self._image_path))
352
353         cmd.append('--test-shell')
354
355         if self._port.get_option('startup_dialog'):
356             cmd.append('--testshell-startup-dialog')
357
358         if self._port.get_option('gp_fault_error_box'):
359             cmd.append('--gp-fault-error-box')
360
361         if self._port.get_option('js_flags') is not None:
362             cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"')
363
364         if self._port.get_option('stress_opt'):
365             cmd.append('--stress-opt')
366
367         if self._port.get_option('stress_deopt'):
368             cmd.append('--stress-deopt')
369
370         if self._port.get_option('accelerated_compositing'):
371             cmd.append('--enable-accelerated-compositing')
372         if self._port.get_option('accelerated_2d_canvas'):
373             cmd.append('--enable-accelerated-2d-canvas')
374         if self._port.get_option('enable_hardware_gpu'):
375             cmd.append('--enable-hardware-gpu')
376         return cmd
377
378     def start(self):
379         # FIXME: Should be an error to call this method twice.
380         cmd = self.cmd_line()
381
382         # We need to pass close_fds=True to work around Python bug #2320
383         # (otherwise we can hang when we kill DumpRenderTree when we are running
384         # multiple threads). See http://bugs.python.org/issue2320 .
385         # Note that close_fds isn't supported on Windows, but this bug only
386         # shows up on Mac and Linux.
387         close_flag = sys.platform not in ('win32', 'cygwin')
388         self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
389                                       stdout=subprocess.PIPE,
390                                       stderr=subprocess.STDOUT,
391                                       close_fds=close_flag)
392
393     def poll(self):
394         # poll() is not threadsafe and can throw OSError due to:
395         # http://bugs.python.org/issue1731717
396         return self._proc.poll()
397
398     def _write_command_and_read_line(self, input=None):
399         """Returns a tuple: (line, did_crash)"""
400         try:
401             if input:
402                 if isinstance(input, unicode):
403                     # DRT expects utf-8
404                     input = input.encode("utf-8")
405                 self._proc.stdin.write(input)
406             # DumpRenderTree text output is always UTF-8.  However some tests
407             # (e.g. webarchive) may spit out binary data instead of text so we
408             # don't bother to decode the output.
409             line = self._proc.stdout.readline()
410             # We could assert() here that line correctly decodes as UTF-8.
411             return (line, False)
412         except IOError, e:
413             _log.error("IOError communicating w/ DRT: " + str(e))
414             return (None, True)
415
416     def _test_shell_command(self, uri, timeoutms, checksum):
417         cmd = uri
418         if timeoutms:
419             cmd += ' ' + str(timeoutms)
420         if checksum:
421             cmd += ' ' + checksum
422         cmd += "\n"
423         return cmd
424
425     def _output_image(self):
426         """Returns the image output which driver generated."""
427         png_path = self._image_path
428         if png_path and self._port._filesystem.isfile(png_path):
429             return self._port._filesystem.read_binary_file(png_path)
430         else:
431             return None
432
433     def _output_image_with_retry(self):
434         # Retry a few more times because open() sometimes fails on Windows,
435         # raising "IOError: [Errno 13] Permission denied:"
436         retry_num = 50
437         timeout_seconds = 5.0
438         for i in range(retry_num):
439             try:
440                 return self._output_image()
441             except IOError, e:
442                 if e.errno == errno.EACCES:
443                     time.sleep(timeout_seconds / retry_num)
444                 else:
445                     raise e
446         return self._output_image()
447
448     def run_test(self, test_input):
449         output = []
450         error = []
451         crash = False
452         timeout = False
453         actual_uri = None
454         actual_checksum = None
455
456         start_time = time.time()
457
458         uri = self._port.filename_to_uri(test_input.filename)
459         cmd = self._test_shell_command(uri, test_input.timeout,
460                                        test_input.image_hash)
461         (line, crash) = self._write_command_and_read_line(input=cmd)
462
463         while not crash and line.rstrip() != "#EOF":
464             # Make sure we haven't crashed.
465             if line == '' and self.poll() is not None:
466                 # This is hex code 0xc000001d, which is used for abrupt
467                 # termination. This happens if we hit ctrl+c from the prompt
468                 # and we happen to be waiting on DRT.
469                 # sdoyon: Not sure for which OS and in what circumstances the
470                 # above code is valid. What works for me under Linux to detect
471                 # ctrl+c is for the subprocess returncode to be negative
472                 # SIGINT. And that agrees with the subprocess documentation.
473                 if (-1073741510 == self._proc.returncode or
474                     - signal.SIGINT == self._proc.returncode):
475                     raise KeyboardInterrupt
476                 crash = True
477                 break
478
479             # Don't include #URL lines in our output
480             if line.startswith("#URL:"):
481                 actual_uri = line.rstrip()[5:]
482                 if uri != actual_uri:
483                     # GURL capitalizes the drive letter of a file URL.
484                     if (not re.search("^file:///[a-z]:", uri) or
485                         uri.lower() != actual_uri.lower()):
486                         _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
487                                    (uri, actual_uri))
488                         raise AssertionError("test out of sync")
489             elif line.startswith("#MD5:"):
490                 actual_checksum = line.rstrip()[5:]
491             elif line.startswith("#TEST_TIMED_OUT"):
492                 timeout = True
493                 # Test timed out, but we still need to read until #EOF.
494             elif actual_uri:
495                 output.append(line)
496             else:
497                 error.append(line)
498
499             (line, crash) = self._write_command_and_read_line(input=None)
500
501         run_time = time.time() - start_time
502         return test_output.TestOutput(
503             ''.join(output), self._output_image_with_retry(), actual_checksum,
504             crash, run_time, timeout, ''.join(error))
505
506     def stop(self):
507         if self._proc:
508             self._proc.stdin.close()
509             self._proc.stdout.close()
510             if self._proc.stderr:
511                 self._proc.stderr.close()
512             if sys.platform not in ('win32', 'cygwin'):
513                 # Closing stdin/stdout/stderr hangs sometimes on OS X,
514                 # (see __init__(), above), and anyway we don't want to hang
515                 # the harness if DRT is buggy, so we wait a couple
516                 # seconds to give DRT a chance to clean up, but then
517                 # force-kill the process if necessary.
518                 KILL_TIMEOUT = 3.0
519                 timeout = time.time() + KILL_TIMEOUT
520                 # poll() is not threadsafe and can throw OSError due to:
521                 # http://bugs.python.org/issue1731717
522                 while self._proc.poll() is None and time.time() < timeout:
523                     time.sleep(0.1)
524                 # poll() is not threadsafe and can throw OSError due to:
525                 # http://bugs.python.org/issue1731717
526                 if self._proc.poll() is None:
527                     _log.warning('stopping test driver timed out, '
528                                  'killing it')
529                     self._port._executive.kill_process(self._proc.pid)