[WK2][GTK] Fix unit test WebKit2APITests/WebKitWebView/mouse-target
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
62
63     SKIPPED = [
64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
69         SkippedTest("unittests/testatk", "/webkit/atk/getTextInParagraphAndBodyModerate", "Test fails", 105538),
70         SkippedTest("WebKit2APITests/TestInspectorServer", SkippedTest.ENTIRE_SUITE, "Test times out", 105866),
71         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
72         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test fails", 100408),
73         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/hide", "Test always fails in Xvfb", 89810),
74         SkippedTest("WebKit2APITests/TestWebKitWebContext", "/webkit2/WebKitWebContext/uri-scheme", "Test fails", 104779),
75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
82         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
83         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
84         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
85     ]
86
87     def __init__(self, options, tests=[]):
88         self._options = options
89         self._build_type = "Debug" if self._options.debug else "Release"
90
91         self._programs_path = common.build_path_for_build_types((self._build_type,), "Programs")
92         self._tests = self._get_tests(tests)
93         self._skipped_tests = TestRunner.SKIPPED
94         if not sys.stdout.isatty():
95             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
96
97         # These SPI daemons need to be active for the accessibility tests to work.
98         self._spi_registryd = None
99         self._spi_bus_launcher = None
100
101     def _get_tests(self, tests):
102         if tests:
103             return tests
104
105         tests = []
106         for test_dir in self.TEST_DIRS:
107             absolute_test_dir = os.path.join(self._programs_path, test_dir)
108             if not os.path.isdir(absolute_test_dir):
109                 continue
110             for test_file in os.listdir(absolute_test_dir):
111                 if not test_file.lower().startswith("test"):
112                     continue
113                 test_path = os.path.join(self._programs_path, test_dir, test_file)
114                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
115                     tests.append(test_path)
116         return tests
117
118     def _lookup_atspi2_binary(self, filename):
119         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
120         if not exec_prefix:
121             return None
122         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
123             filepath = os.path.join(exec_prefix, path, filename)
124             if os.path.isfile(filepath):
125                 return filepath
126
127         return None
128
129     def _start_accessibility_daemons(self):
130         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
131         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
132         if not spi_bus_launcher_path or not spi_registryd_path:
133             return False
134
135         try:
136             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
137         except:
138             sys.stderr.write("Failed to launch the accessibility bus\n")
139             sys.stderr.flush()
140             return False
141
142         # We need to wait until the SPI bus is launched before trying to start the SPI
143         # registry, so we spin a main loop until the bus name appears on DBus.
144         loop = GLib.MainLoop()
145         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
146                            lambda *args: loop.quit(), None)
147         loop.run()
148
149         try:
150             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
151         except:
152             sys.stderr.write("Failed to launch the accessibility registry\n")
153             sys.stderr.flush()
154             return False
155
156         return True
157
158     def _setup_testing_environment(self):
159         self._test_env = os.environ
160         self._test_env["DISPLAY"] = self._options.display
161         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
162         self._test_env['GSETTINGS_BACKEND'] = 'memory'
163         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
164         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path_for_build_types((self._build_type,), "Libraries")
165         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
166
167         try:
168             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
169                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
170         except Exception as e:
171             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
172             sys.stderr.flush()
173             return False
174
175         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
176         if not self._start_accessibility_daemons():
177             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
178             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
179         return True
180
181     def _tear_down_testing_environment(self):
182         if self._spi_registryd:
183             self._spi_registryd.terminate()
184         if self._spi_bus_launcher:
185             self._spi_bus_launcher.terminate()
186         self._xvfb.terminate()
187
188     def _test_cases_to_skip(self, test_program):
189         if self._options.skipped_action != 'skip':
190             return []
191
192         test_cases = []
193         for skipped in self._skipped_tests:
194             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
195                 test_cases.append(skipped.test_case)
196         return test_cases
197
198     def _should_run_test_program(self, test_program):
199         # This is not affected by the command-line arguments, since programs are skipped for
200         # problems in the harness, such as failing to start the accessibility bus.
201         for skipped in self._skipped_tests:
202             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
203                 return False
204         return True
205
206     def _get_child_pid_from_test_output(self, output):
207         if not output:
208             return -1
209         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
210         if not match:
211             return -1
212         return int(match.group('child_pid'))
213
214     def _kill_process(self, pid):
215         try:
216             os.kill(pid, SIGKILL)
217         except OSError:
218             # Process already died.
219             pass
220
221     def _run_test_command(self, command, timeout=-1):
222         def alarm_handler(signum, frame):
223             raise TestTimeout
224
225         child_pid = [-1]
226         def parse_line(line, child_pid = child_pid):
227             if child_pid[0] == -1:
228                 child_pid[0] = self._get_child_pid_from_test_output(line)
229
230             if sys.stdout.isatty():
231                 sys.stdout.write(line)
232             else:
233                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
234
235         def waitpid(pid):
236             while True:
237                 try:
238                     return os.waitpid(pid, 0)
239                 except (OSError, IOError) as e:
240                     if e.errno == errno.EINTR:
241                         continue
242                     raise
243
244         def return_code_from_exit_status(status):
245             if os.WIFSIGNALED(status):
246                 return -os.WTERMSIG(status)
247             elif os.WIFEXITED(status):
248                 return os.WEXITSTATUS(status)
249             else:
250                 # Should never happen
251                 raise RuntimeError("Unknown child exit status!")
252
253         pid, fd = os.forkpty()
254         if pid == 0:
255             os.execvpe(command[0], command, self._test_env)
256             sys.exit(0)
257
258         if timeout > 0:
259             signal(SIGALRM, alarm_handler)
260             alarm(timeout)
261
262         try:
263             common.parse_output_lines(fd, parse_line)
264             if timeout > 0:
265                 alarm(0)
266         except TestTimeout:
267             self._kill_process(pid)
268             if child_pid[0] > 0:
269                 self._kill_process(child_pid[0])
270             raise
271
272         try:
273             dummy, status = waitpid(pid)
274         except OSError as e:
275             if e.errno != errno.ECHILD:
276                 raise
277             # This happens if SIGCLD is set to be ignored or waiting
278             # for child processes has otherwise been disabled for our
279             # process.  This child is dead, we can't get the status.
280             status = 0
281
282         return not return_code_from_exit_status(status)
283
284     def _run_test_glib(self, test_program):
285         tester_command = ['gtester']
286         if self._options.verbose:
287             tester_command.append('--verbose')
288         for test_case in self._test_cases_to_skip(test_program):
289             tester_command.extend(['-s', test_case])
290         tester_command.append(test_program)
291
292         return self._run_test_command(tester_command, self._options.timeout)
293
294     def _run_test_google(self, test_program):
295         tester_command = [test_program]
296         skipped_tests_cases = self._test_cases_to_skip(test_program)
297         if skipped_tests_cases:
298             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
299
300         return self._run_test_command(tester_command, self._options.timeout)
301
302     def _run_test(self, test_program):
303         if "unittests" in test_program or "WebKit2APITests" in test_program:
304             return self._run_test_glib(test_program)
305
306         if "TestWebKitAPI" in test_program:
307             return self._run_test_google(test_program)
308
309         return False
310
311     def run_tests(self):
312         if not self._tests:
313             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
314             sys.stderr.flush()
315             return 1
316
317         if not self._setup_testing_environment():
318             return 1
319
320         # Remove skipped tests now instead of when we find them, because
321         # some tests might be skipped while setting up the test environment.
322         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
323
324         failed_tests = []
325         timed_out_tests = []
326         try:
327             for test in self._tests:
328                 success = True
329                 try:
330                     success = self._run_test(test)
331                 except TestTimeout:
332                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
333                     sys.stdout.flush()
334                     timed_out_tests.append(test)
335
336                 if not success:
337                     failed_tests.append(test)
338         finally:
339             self._tear_down_testing_environment()
340
341         if failed_tests:
342             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
343             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
344             sys.stdout.flush()
345
346         if timed_out_tests:
347             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
348             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
349             sys.stdout.flush()
350
351         if self._skipped_tests and self._options.skipped_action == 'skip':
352             sys.stdout.write("Tests skipped (%d):\n%s\n" %
353                              (len(self._skipped_tests),
354                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
355             sys.stdout.flush()
356
357         return len(failed_tests) + len(timed_out_tests)
358
359 if __name__ == "__main__":
360     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
361         print "***"
362         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
363         print "***"
364
365     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
366     option_parser.add_option('-r', '--release',
367                              action='store_true', dest='release',
368                              help='Run in Release')
369     option_parser.add_option('-d', '--debug',
370                              action='store_true', dest='debug',
371                              help='Run in Debug')
372     option_parser.add_option('-v', '--verbose',
373                              action='store_true', dest='verbose',
374                              help='Run gtester in verbose mode')
375     option_parser.add_option('--display', action='store', dest='display', default=':55',
376                              help='Display to run Xvfb')
377     option_parser.add_option('--skipped', action='store', dest='skipped_action',
378                              choices=['skip', 'ignore', 'only'], default='skip',
379                              metavar='skip|ignore|only',
380                              help='Specifies how to treat the skipped tests')
381     option_parser.add_option('-t', '--timeout',
382                              action='store', type='int', dest='timeout', default=10,
383                              help='Time in seconds until a test times out')
384     options, args = option_parser.parse_args()
385
386     sys.exit(TestRunner(options, args).run_tests())