3634e5b1cbcfdc7de922db314857cd9815ab500b
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
62
63     SKIPPED = [
64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
69         SkippedTest("unittests/testatk", "/webkit/atk/getTextInParagraphAndBodyModerate", "Test fails", 105538),
70         SkippedTest("WebKit2APITests/TestInspectorServer", SkippedTest.ENTIRE_SUITE, "Test times out", 105866),
71         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
72         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test fails", 100408),
73         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/hide", "Test always fails in Xvfb", 89810),
74         SkippedTest("WebKit2APITests/TestWebKitWebContext", "/webkit2/WebKitWebContext/uri-scheme", "Test fails", 104779),
75         SkippedTest("WebKit2APITests/TestWebKitWebView", "/webkit2/WebKitWebView/mouse-target", "Test is flaky in GTK Linux 32-bit Release bot", 82866),
76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
82         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
83         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
84         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
85         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
86     ]
87
88     def __init__(self, options, tests=[]):
89         self._options = options
90         self._build_type = "Debug" if self._options.debug else "Release"
91
92         self._programs_path = common.build_path_for_build_types((self._build_type,), "Programs")
93         self._tests = self._get_tests(tests)
94         self._skipped_tests = TestRunner.SKIPPED
95         if not sys.stdout.isatty():
96             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
97
98         # These SPI daemons need to be active for the accessibility tests to work.
99         self._spi_registryd = None
100         self._spi_bus_launcher = None
101
102     def _get_tests(self, tests):
103         if tests:
104             return tests
105
106         tests = []
107         for test_dir in self.TEST_DIRS:
108             absolute_test_dir = os.path.join(self._programs_path, test_dir)
109             if not os.path.isdir(absolute_test_dir):
110                 continue
111             for test_file in os.listdir(absolute_test_dir):
112                 if not test_file.lower().startswith("test"):
113                     continue
114                 test_path = os.path.join(self._programs_path, test_dir, test_file)
115                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
116                     tests.append(test_path)
117         return tests
118
119     def _lookup_atspi2_binary(self, filename):
120         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
121         if not exec_prefix:
122             return None
123         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
124             filepath = os.path.join(exec_prefix, path, filename)
125             if os.path.isfile(filepath):
126                 return filepath
127
128         return None
129
130     def _start_accessibility_daemons(self):
131         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
132         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
133         if not spi_bus_launcher_path or not spi_registryd_path:
134             return False
135
136         try:
137             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
138         except:
139             sys.stderr.write("Failed to launch the accessibility bus\n")
140             sys.stderr.flush()
141             return False
142
143         # We need to wait until the SPI bus is launched before trying to start the SPI
144         # registry, so we spin a main loop until the bus name appears on DBus.
145         loop = GLib.MainLoop()
146         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
147                            lambda *args: loop.quit(), None)
148         loop.run()
149
150         try:
151             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
152         except:
153             sys.stderr.write("Failed to launch the accessibility registry\n")
154             sys.stderr.flush()
155             return False
156
157         return True
158
159     def _setup_testing_environment(self):
160         self._test_env = os.environ
161         self._test_env["DISPLAY"] = self._options.display
162         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
163         self._test_env['GSETTINGS_BACKEND'] = 'memory'
164         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
165         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path_for_build_types((self._build_type,), "Libraries")
166         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
167
168         try:
169             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
170                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
171         except Exception as e:
172             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
173             sys.stderr.flush()
174             return False
175
176         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
177         if not self._start_accessibility_daemons():
178             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
179             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
180         return True
181
182     def _tear_down_testing_environment(self):
183         if self._spi_registryd:
184             self._spi_registryd.terminate()
185         if self._spi_bus_launcher:
186             self._spi_bus_launcher.terminate()
187         self._xvfb.terminate()
188
189     def _test_cases_to_skip(self, test_program):
190         if self._options.skipped_action != 'skip':
191             return []
192
193         test_cases = []
194         for skipped in self._skipped_tests:
195             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
196                 test_cases.append(skipped.test_case)
197         return test_cases
198
199     def _should_run_test_program(self, test_program):
200         # This is not affected by the command-line arguments, since programs are skipped for
201         # problems in the harness, such as failing to start the accessibility bus.
202         for skipped in self._skipped_tests:
203             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
204                 return False
205         return True
206
207     def _get_child_pid_from_test_output(self, output):
208         if not output:
209             return -1
210         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
211         if not match:
212             return -1
213         return int(match.group('child_pid'))
214
215     def _kill_process(self, pid):
216         try:
217             os.kill(pid, SIGKILL)
218         except OSError:
219             # Process already died.
220             pass
221
222     def _run_test_command(self, command, timeout=-1):
223         def alarm_handler(signum, frame):
224             raise TestTimeout
225
226         child_pid = [-1]
227         def parse_line(line, child_pid = child_pid):
228             if child_pid[0] == -1:
229                 child_pid[0] = self._get_child_pid_from_test_output(line)
230
231             if sys.stdout.isatty():
232                 sys.stdout.write(line)
233             else:
234                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
235
236         def waitpid(pid):
237             while True:
238                 try:
239                     return os.waitpid(pid, 0)
240                 except (OSError, IOError) as e:
241                     if e.errno == errno.EINTR:
242                         continue
243                     raise
244
245         def return_code_from_exit_status(status):
246             if os.WIFSIGNALED(status):
247                 return -os.WTERMSIG(status)
248             elif os.WIFEXITED(status):
249                 return os.WEXITSTATUS(status)
250             else:
251                 # Should never happen
252                 raise RuntimeError("Unknown child exit status!")
253
254         pid, fd = os.forkpty()
255         if pid == 0:
256             os.execvpe(command[0], command, self._test_env)
257             sys.exit(0)
258
259         if timeout > 0:
260             signal(SIGALRM, alarm_handler)
261             alarm(timeout)
262
263         try:
264             common.parse_output_lines(fd, parse_line)
265             if timeout > 0:
266                 alarm(0)
267         except TestTimeout:
268             self._kill_process(pid)
269             if child_pid[0] > 0:
270                 self._kill_process(child_pid[0])
271             raise
272
273         try:
274             dummy, status = waitpid(pid)
275         except OSError as e:
276             if e.errno != errno.ECHILD:
277                 raise
278             # This happens if SIGCLD is set to be ignored or waiting
279             # for child processes has otherwise been disabled for our
280             # process.  This child is dead, we can't get the status.
281             status = 0
282
283         return not return_code_from_exit_status(status)
284
285     def _run_test_glib(self, test_program):
286         tester_command = ['gtester']
287         if self._options.verbose:
288             tester_command.append('--verbose')
289         for test_case in self._test_cases_to_skip(test_program):
290             tester_command.extend(['-s', test_case])
291         tester_command.append(test_program)
292
293         return self._run_test_command(tester_command, self._options.timeout)
294
295     def _run_test_google(self, test_program):
296         tester_command = [test_program]
297         skipped_tests_cases = self._test_cases_to_skip(test_program)
298         if skipped_tests_cases:
299             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
300
301         return self._run_test_command(tester_command, self._options.timeout)
302
303     def _run_test(self, test_program):
304         if "unittests" in test_program or "WebKit2APITests" in test_program:
305             return self._run_test_glib(test_program)
306
307         if "TestWebKitAPI" in test_program:
308             return self._run_test_google(test_program)
309
310         return False
311
312     def run_tests(self):
313         if not self._tests:
314             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
315             sys.stderr.flush()
316             return 1
317
318         if not self._setup_testing_environment():
319             return 1
320
321         # Remove skipped tests now instead of when we find them, because
322         # some tests might be skipped while setting up the test environment.
323         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
324
325         failed_tests = []
326         timed_out_tests = []
327         try:
328             for test in self._tests:
329                 success = True
330                 try:
331                     success = self._run_test(test)
332                 except TestTimeout:
333                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
334                     sys.stdout.flush()
335                     timed_out_tests.append(test)
336
337                 if not success:
338                     failed_tests.append(test)
339         finally:
340             self._tear_down_testing_environment()
341
342         if failed_tests:
343             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
344             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
345             sys.stdout.flush()
346
347         if timed_out_tests:
348             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
349             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
350             sys.stdout.flush()
351
352         if self._skipped_tests and self._options.skipped_action == 'skip':
353             sys.stdout.write("Tests skipped (%d):\n%s\n" %
354                              (len(self._skipped_tests),
355                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
356             sys.stdout.flush()
357
358         return len(failed_tests) + len(timed_out_tests)
359
360 if __name__ == "__main__":
361     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
362         print "***"
363         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
364         print "***"
365
366     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
367     option_parser.add_option('-r', '--release',
368                              action='store_true', dest='release',
369                              help='Run in Release')
370     option_parser.add_option('-d', '--debug',
371                              action='store_true', dest='debug',
372                              help='Run in Debug')
373     option_parser.add_option('-v', '--verbose',
374                              action='store_true', dest='verbose',
375                              help='Run gtester in verbose mode')
376     option_parser.add_option('--display', action='store', dest='display', default=':55',
377                              help='Display to run Xvfb')
378     option_parser.add_option('--skipped', action='store', dest='skipped_action',
379                              choices=['skip', 'ignore', 'only'], default='skip',
380                              metavar='skip|ignore|only',
381                              help='Specifies how to treat the skipped tests')
382     option_parser.add_option('-t', '--timeout',
383                              action='store', type='int', dest='timeout', default=10,
384                              help='Time in seconds until a test times out')
385     options, args = option_parser.parse_args()
386
387     sys.exit(TestRunner(options, args).run_tests())