37939ca25ad51f743b76dac403f209ebb32ca57a
[WebKit-https.git] / Tools / gtk / run-api-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import common
21 import subprocess
22 import os
23 import sys
24 import optparse
25 import re
26 from signal import alarm, signal, SIGALRM, SIGKILL
27 from gi.repository import Gio, GLib
28
29 class SkippedTest:
30     def __init__(self, test, reason, bug=None, test_cases=[]):
31         self.test = test
32         self.reason = reason
33         self.bug = bug
34         self.test_cases = test_cases
35
36     def __str__(self):
37         skipped_test_str = "%s" % self.test
38         if self.test_cases:
39             skipped_test_str += " [%s]" % ", ".join(self.test_cases)
40         skipped_test_str += ": %s " % self.reason
41         if self.bug is not None:
42             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
43         return skipped_test_str
44
45 class TestTimeout(Exception):
46     pass
47
48 class TestRunner:
49
50     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI/WTF", "TestWebKitAPI/WebKit2" ]
51
52     SKIPPED = [
53         SkippedTest("unittests/testdownload",
54                     "Test fails in GTK Linux 64-bit Release bot",
55                     82329,
56                     ["/webkit/download/not-found"]),
57         SkippedTest("unittests/testwebview",
58                     "Test times out in GTK Linux 64-bit Release bot",
59                     82328,
60                     ["/webkit/webview/icon-uri"]),
61         SkippedTest("unittests/testwebresource",
62                     "Test fails in GTK Linux 64-bit Release bot",
63                     82330,
64                     ["/webkit/webresource/sub_resource_loading"]),
65         SkippedTest("unittests/testwebinspector",
66                     "Test is flaky in GTK Linux 32-bit Release bot",
67                     82869,
68                     ["/webkit/webinspector/close-and-inspect"]),
69         SkippedTest("WebKit2APITests/TestWebKitWebView",
70                     "Test is flaky in GTK Linux 32-bit Release bot",
71                     82866,
72                     ["/webkit2/WebKitWebView/mouse-target"]),
73         SkippedTest("WebKit2APITests/TestResources",
74                     "Test is flaky in GTK Linux 32-bit Release bot",
75                     82868,
76                     ["/webkit2/WebKitWebView/resources"]),
77         SkippedTest("TestWebKitAPI/WebKit2/TestWKConnection",
78                     "Test times out",
79                     84959),
80         SkippedTest("TestWebKitAPI/WebKit2/TestRestoreSessionStateContainingFormData",
81                     "Session State is not implemented in GTK+ port",
82                     84960),
83         SkippedTest("TestWebKitAPI/WebKit2/TestSpacebarScrolling",
84                     "Test fails",
85                     84961),
86         SkippedTest("TestWebKitAPI/WebKit2/TestNewFirstVisuallyNonEmptyLayoutFrames",
87                     "Test fails",
88                     85037),
89         SkippedTest("TestWebKitAPI/WebKit2/TestMouseMoveAfterCrash",
90                     "Test is flaky",
91                     85066),
92         SkippedTest("TestWebKitAPI/WTF/TestHashMap",
93                     "Test fails",
94                     88419)
95     ]
96
97     def __init__(self, options, tests=[]):
98         self._options = options
99         self._programs_path = common.build_path("Programs")
100         self._tests = self._get_tests(tests)
101         self._skipped_tests = TestRunner.SKIPPED
102
103         # These SPI daemons need to be active for the accessibility tests to work.
104         self._spi_registryd = None
105         self._spi_bus_launcher = None
106
107     def _get_tests(self, tests):
108         if tests:
109             return tests
110
111         tests = []
112         for test_dir in self.TEST_DIRS:
113             absolute_test_dir = os.path.join(self._programs_path, test_dir)
114             if not os.path.isdir(absolute_test_dir):
115                 continue
116             for test_file in os.listdir(absolute_test_dir):
117                 if not test_file.lower().startswith("test"):
118                     continue
119                 test_path = os.path.join(self._programs_path, test_dir, test_file)
120                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
121                     tests.append(test_path)
122         return tests
123
124     def _lookup_atspi2_binary(self, filename):
125         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
126         if not exec_prefix:
127             return None
128         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
129             filepath = os.path.join(exec_prefix, path, filename)
130             if os.path.isfile(filepath):
131                 return filepath
132
133         return None
134
135     def _start_accessibility_daemons(self):
136         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
137         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
138         if not spi_bus_launcher_path or not spi_registryd_path:
139             return False
140
141         try:
142             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
143         except:
144             sys.stderr.write("Failed to launch the accessibility bus\n")
145             sys.stderr.flush()
146             return False
147
148         # We need to wait until the SPI bus is launched before trying to start the SPI
149         # registry, so we spin a main loop until the bus name appears on DBus.
150         loop = GLib.MainLoop()
151         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
152                            lambda *args: loop.quit(), None)
153         loop.run()
154
155         try:
156             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
157         except:
158             sys.stderr.write("Failed to launch the accessibility registry\n")
159             sys.stderr.flush()
160             return False
161
162         return True
163
164     def _setup_testing_environment(self):
165         self._test_env = os.environ
166         self._test_env["DISPLAY"] = self._options.display
167         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
168         self._test_env['GSETTINGS_BACKEND'] = 'memory'
169         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
170         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path("Libraries")
171         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
172
173         try:
174             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
175                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
176         except Exception as e:
177             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
178             sys.stderr.flush()
179             return False
180
181         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
182         if not self._start_accessibility_daemons():
183             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
184             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility",
185                                                    "Could not start accessibility bus"))
186         return True
187
188     def _tear_down_testing_environment(self):
189         if self._spi_registryd:
190             self._spi_registryd.terminate()
191         if self._spi_bus_launcher:
192             self._spi_bus_launcher.terminate()
193         self._xvfb.terminate()
194
195     def _find_skipped_test(self, test):
196         for skipped in self._skipped_tests:
197             if test.endswith(skipped.test):
198                 return skipped
199         return None
200
201     def _test_cases_to_skip(self, test):
202         if self._options.skipped_action != 'skip':
203             return []
204
205         skipped = self._find_skipped_test(test)
206         if skipped is not None:
207             return skipped.test_cases
208         return []
209
210     def _should_run_test(self, test):
211         # Skipped test are ignored, run all tests.
212         if self._options.skipped_action == 'ignore':
213             return True
214
215         skipped = self._find_skipped_test(test)
216         # By default skipped test are skipped, run them only when there are specific test cases failing.
217         if self._options.skipped_action == 'skip':
218             return skipped is None or skipped.test_cases
219
220         # Run only skipped tests.
221         return skipped is not None
222
223     def _get_child_pid_from_test_output(self, output):
224         if not output:
225             return -1
226         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
227         if not match:
228             return -1
229         return int(match.group('child_pid'))
230
231     def _kill_process(self, pid):
232         try:
233             os.kill(pid, SIGKILL)
234         except OSError:
235             # Process already died.
236             pass
237
238     def _run_test_command(self, command, timeout=-1):
239         def alarm_handler(signum, frame):
240             raise TestTimeout
241
242         p = subprocess.Popen(command, stdout=subprocess.PIPE, env=self._test_env)
243         if timeout > 0:
244             signal(SIGALRM, alarm_handler)
245             alarm(timeout)
246
247         stdout = ""
248         try:
249             stdout = p.communicate()[0]
250             if timeout > 0:
251                 alarm(0)
252             sys.stdout.write(stdout)
253             sys.stdout.flush()
254         except TestTimeout:
255             self._kill_process(p.pid)
256             child_pid = self._get_child_pid_from_test_output(stdout)
257             if child_pid > 0:
258                 self._kill_process(child_pid)
259             raise
260
261         return not p.returncode
262
263     def _run_test_glib(self, test):
264         tester_command = ['gtester']
265         if self._options.verbose:
266             tester_command.append('--verbose')
267         for test_case in self._test_cases_to_skip(test):
268             tester_command.extend(['-s', test_case])
269         tester_command.append(test)
270
271         return self._run_test_command(tester_command, self._options.timeout)
272
273     def _run_test_google(self, test):
274         tester_command = [test, "--gtest_throw_on_failure"]
275         skipped_tests_cases = self._test_cases_to_skip(test)
276         if skipped_tests_cases:
277             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
278
279         return self._run_test_command(tester_command, self._options.timeout)
280
281     def _run_test(self, test):
282         if "unittests" in test or "WebKit2APITests" in test:
283             return self._run_test_glib(test)
284
285         if "TestWebKitAPI" in test:
286             return self._run_test_google(test)
287
288         return False
289
290     def run_tests(self):
291         if not self._tests:
292             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
293             sys.stderr.flush()
294             return 1
295
296         if not self._setup_testing_environment():
297             return 1
298
299         # Remove skipped tests now instead of when we find them, because
300         # some tests might be skipped while setting up the test environment.
301         self._tests = [test for test in self._tests if self._should_run_test(test)]
302
303         failed_tests = []
304         timed_out_tests = []
305         try:
306             for test in self._tests:
307                 success = True
308                 try:
309                     success = self._run_test(test)
310                 except TestTimeout:
311                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
312                     sys.stdout.flush()
313                     timed_out_tests.append(test)
314
315                 if not success:
316                     failed_tests.append(test)
317         finally:
318             self._tear_down_testing_environment()
319
320         if failed_tests:
321             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
322             sys.stdout.write("Tests failed: %s\n" % ", ".join(names))
323             sys.stdout.flush()
324
325         if timed_out_tests:
326             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
327             sys.stdout.write("Tests that timed out: %s\n" % ", ".join(names))
328             sys.stdout.flush()
329
330         if self._skipped_tests and self._options.skipped_action == 'skip':
331             sys.stdout.write("Tests skipped:\n%s\n" % "\n".join([str(skipped) for skipped in self._skipped_tests]))
332             sys.stdout.flush()
333
334         return len(failed_tests)
335
336 if __name__ == "__main__":
337     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
338     option_parser.add_option('-r', '--release',
339                              action='store_true', dest='release',
340                              help='Run in Release')
341     option_parser.add_option('-d', '--debug',
342                              action='store_true', dest='debug',
343                              help='Run in Debug')
344     option_parser.add_option('-v', '--verbose',
345                              action='store_true', dest='verbose',
346                              help='Run gtester in verbose mode')
347     option_parser.add_option('--display', action='store', dest='display', default=':55',
348                              help='Display to run Xvfb')
349     option_parser.add_option('--skipped', action='store', dest='skipped_action',
350                              choices=['skip', 'ignore', 'only'], default='skip',
351                              metavar='skip|ignore|only',
352                              help='Specifies how to treat the skipped tests')
353     option_parser.add_option('-t', '--timeout',
354                              action='store', type='int', dest='timeout', default=10,
355                              help='Time in seconds until a test times out')
356     options, args = option_parser.parse_args()
357
358     sys.exit(TestRunner(options, args).run_tests())