Unreviewed. Add Silvia Pfeiffer to contributor list.
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
62
63     SKIPPED = [
64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
69         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
70         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test fails", 100408),
71         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/hide", "Test always fails in Xvfb", 89810),
72         SkippedTest("WebKit2APITests/TestWebKitWebContext", "/webkit2/WebKitWebContext/uri-scheme", "Test fails", 104779),
73         SkippedTest("WebKit2APITests/TestWebKitWebView", "/webkit2/WebKitWebView/mouse-target", "Test is flaky in GTK Linux 32-bit Release bot", 82866),
74         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
82     ]
83
84     def __init__(self, options, tests=[]):
85         self._options = options
86         self._programs_path = common.build_path("Programs")
87         self._tests = self._get_tests(tests)
88         self._skipped_tests = TestRunner.SKIPPED
89         if not sys.stdout.isatty():
90             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
91
92         # These SPI daemons need to be active for the accessibility tests to work.
93         self._spi_registryd = None
94         self._spi_bus_launcher = None
95
96     def _get_tests(self, tests):
97         if tests:
98             return tests
99
100         tests = []
101         for test_dir in self.TEST_DIRS:
102             absolute_test_dir = os.path.join(self._programs_path, test_dir)
103             if not os.path.isdir(absolute_test_dir):
104                 continue
105             for test_file in os.listdir(absolute_test_dir):
106                 if not test_file.lower().startswith("test"):
107                     continue
108                 test_path = os.path.join(self._programs_path, test_dir, test_file)
109                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
110                     tests.append(test_path)
111         return tests
112
113     def _lookup_atspi2_binary(self, filename):
114         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
115         if not exec_prefix:
116             return None
117         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
118             filepath = os.path.join(exec_prefix, path, filename)
119             if os.path.isfile(filepath):
120                 return filepath
121
122         return None
123
124     def _start_accessibility_daemons(self):
125         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
126         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
127         if not spi_bus_launcher_path or not spi_registryd_path:
128             return False
129
130         try:
131             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
132         except:
133             sys.stderr.write("Failed to launch the accessibility bus\n")
134             sys.stderr.flush()
135             return False
136
137         # We need to wait until the SPI bus is launched before trying to start the SPI
138         # registry, so we spin a main loop until the bus name appears on DBus.
139         loop = GLib.MainLoop()
140         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
141                            lambda *args: loop.quit(), None)
142         loop.run()
143
144         try:
145             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
146         except:
147             sys.stderr.write("Failed to launch the accessibility registry\n")
148             sys.stderr.flush()
149             return False
150
151         return True
152
153     def _setup_testing_environment(self):
154         self._test_env = os.environ
155         self._test_env["DISPLAY"] = self._options.display
156         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
157         self._test_env['GSETTINGS_BACKEND'] = 'memory'
158         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
159         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path("Libraries")
160         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
161
162         try:
163             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
164                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
165         except Exception as e:
166             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
167             sys.stderr.flush()
168             return False
169
170         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
171         if not self._start_accessibility_daemons():
172             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
173             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
174         return True
175
176     def _tear_down_testing_environment(self):
177         if self._spi_registryd:
178             self._spi_registryd.terminate()
179         if self._spi_bus_launcher:
180             self._spi_bus_launcher.terminate()
181         self._xvfb.terminate()
182
183     def _test_cases_to_skip(self, test_program):
184         if self._options.skipped_action != 'skip':
185             return []
186
187         test_cases = []
188         for skipped in self._skipped_tests:
189             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
190                 test_cases.append(skipped.test_case)
191         return test_cases
192
193     def _should_run_test_program(self, test_program):
194         # This is not affected by the command-line arguments, since programs are skipped for
195         # problems in the harness, such as failing to start the accessibility bus.
196         for skipped in self._skipped_tests:
197             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
198                 return False
199         return True
200
201     def _get_child_pid_from_test_output(self, output):
202         if not output:
203             return -1
204         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
205         if not match:
206             return -1
207         return int(match.group('child_pid'))
208
209     def _kill_process(self, pid):
210         try:
211             os.kill(pid, SIGKILL)
212         except OSError:
213             # Process already died.
214             pass
215
216     def _run_test_command(self, command, timeout=-1):
217         def alarm_handler(signum, frame):
218             raise TestTimeout
219
220         child_pid = [-1]
221         def parse_line(line, child_pid = child_pid):
222             if child_pid[0] == -1:
223                 child_pid[0] = self._get_child_pid_from_test_output(line)
224
225             if sys.stdout.isatty():
226                 sys.stdout.write(line)
227             else:
228                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
229
230         def waitpid(pid):
231             while True:
232                 try:
233                     return os.waitpid(pid, 0)
234                 except (OSError, IOError) as e:
235                     if e.errno == errno.EINTR:
236                         continue
237                     raise
238
239         def return_code_from_exit_status(status):
240             if os.WIFSIGNALED(status):
241                 return -os.WTERMSIG(status)
242             elif os.WIFEXITED(status):
243                 return os.WEXITSTATUS(status)
244             else:
245                 # Should never happen
246                 raise RuntimeError("Unknown child exit status!")
247
248         pid, fd = os.forkpty()
249         if pid == 0:
250             os.execvpe(command[0], command, self._test_env)
251             sys.exit(0)
252
253         if timeout > 0:
254             signal(SIGALRM, alarm_handler)
255             alarm(timeout)
256
257         try:
258             common.parse_output_lines(fd, parse_line)
259             if timeout > 0:
260                 alarm(0)
261         except TestTimeout:
262             self._kill_process(pid)
263             if child_pid[0] > 0:
264                 self._kill_process(child_pid[0])
265             raise
266
267         try:
268             dummy, status = waitpid(pid)
269         except OSError as e:
270             if e.errno != errno.ECHILD:
271                 raise
272             # This happens if SIGCLD is set to be ignored or waiting
273             # for child processes has otherwise been disabled for our
274             # process.  This child is dead, we can't get the status.
275             status = 0
276
277         return not return_code_from_exit_status(status)
278
279     def _run_test_glib(self, test_program):
280         tester_command = ['gtester']
281         if self._options.verbose:
282             tester_command.append('--verbose')
283         for test_case in self._test_cases_to_skip(test_program):
284             tester_command.extend(['-s', test_case])
285         tester_command.append(test_program)
286
287         return self._run_test_command(tester_command, self._options.timeout)
288
289     def _run_test_google(self, test_program):
290         tester_command = [test_program]
291         skipped_tests_cases = self._test_cases_to_skip(test_program)
292         if skipped_tests_cases:
293             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
294
295         return self._run_test_command(tester_command, self._options.timeout)
296
297     def _run_test(self, test_program):
298         if "unittests" in test_program or "WebKit2APITests" in test_program:
299             return self._run_test_glib(test_program)
300
301         if "TestWebKitAPI" in test_program:
302             return self._run_test_google(test_program)
303
304         return False
305
306     def run_tests(self):
307         if not self._tests:
308             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
309             sys.stderr.flush()
310             return 1
311
312         if not self._setup_testing_environment():
313             return 1
314
315         # Remove skipped tests now instead of when we find them, because
316         # some tests might be skipped while setting up the test environment.
317         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
318
319         failed_tests = []
320         timed_out_tests = []
321         try:
322             for test in self._tests:
323                 success = True
324                 try:
325                     success = self._run_test(test)
326                 except TestTimeout:
327                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
328                     sys.stdout.flush()
329                     timed_out_tests.append(test)
330
331                 if not success:
332                     failed_tests.append(test)
333         finally:
334             self._tear_down_testing_environment()
335
336         if failed_tests:
337             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
338             sys.stdout.write("Tests failed: %s\n" % ", ".join(names))
339             sys.stdout.flush()
340
341         if timed_out_tests:
342             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
343             sys.stdout.write("Tests that timed out: %s\n" % ", ".join(names))
344             sys.stdout.flush()
345
346         if self._skipped_tests and self._options.skipped_action == 'skip':
347             sys.stdout.write("Tests skipped:\n%s\n" % "\n".join([str(skipped) for skipped in self._skipped_tests]))
348             sys.stdout.flush()
349
350         return len(failed_tests)
351
352 if __name__ == "__main__":
353     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
354         print "***"
355         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
356         print "***"
357
358     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
359     option_parser.add_option('-r', '--release',
360                              action='store_true', dest='release',
361                              help='Run in Release')
362     option_parser.add_option('-d', '--debug',
363                              action='store_true', dest='debug',
364                              help='Run in Debug')
365     option_parser.add_option('-v', '--verbose',
366                              action='store_true', dest='verbose',
367                              help='Run gtester in verbose mode')
368     option_parser.add_option('--display', action='store', dest='display', default=':55',
369                              help='Display to run Xvfb')
370     option_parser.add_option('--skipped', action='store', dest='skipped_action',
371                              choices=['skip', 'ignore', 'only'], default='skip',
372                              metavar='skip|ignore|only',
373                              help='Specifies how to treat the skipped tests')
374     option_parser.add_option('-t', '--timeout',
375                              action='store', type='int', dest='timeout', default=10,
376                              help='Time in seconds until a test times out')
377     options, args = option_parser.parse_args()
378
379     sys.exit(TestRunner(options, args).run_tests())