[GTK] Combine WebKit API tests into fewer binaries
[WebKit-https.git] / Tools / gtk / run-api-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import common
21 import subprocess
22 import os
23 import sys
24 import optparse
25 import re
26 from signal import alarm, signal, SIGALRM, SIGKILL
27 from gi.repository import Gio, GLib
28
29 class SkippedTest:
30     ENTIRE_SUITE = None
31
32     def __init__(self, test, test_case, reason, bug=None):
33         self.test = test
34         self.test_case = test_case
35         self.reason = reason
36         self.bug = bug
37
38     def __str__(self):
39         skipped_test_str = "%s" % self.test
40
41         if not(self.skip_entire_suite()):
42             skipped_test_str += " [%s]" % self.test_case
43
44         skipped_test_str += ": %s " % self.reason
45         if self.bug is not None:
46             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
47         return skipped_test_str
48
49     def skip_entire_suite(self):
50         return self.test_case == SkippedTest.ENTIRE_SUITE
51
52 class TestTimeout(Exception):
53     pass
54
55 class TestRunner:
56     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
57
58     SKIPPED = [
59         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
60         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
61         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
62         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
63         SkippedTest("WebKit2APITests/TestWebKitWebView", "/webkit2/WebKitWebView/mouse-target", "Test is flaky in GTK Linux 32-bit Release bot", 82866),
64         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
65         SkippedTest("TestWebKitAPI/TestWTF", "WTF.HashMap", "Test fails", 88419),
66         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
67         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
68         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
69         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
70         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
71         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
72         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
73         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
74     ]
75
76     def __init__(self, options, tests=[]):
77         self._options = options
78         self._programs_path = common.build_path("Programs")
79         self._tests = self._get_tests(tests)
80         self._skipped_tests = TestRunner.SKIPPED
81
82         # These SPI daemons need to be active for the accessibility tests to work.
83         self._spi_registryd = None
84         self._spi_bus_launcher = None
85
86     def _get_tests(self, tests):
87         if tests:
88             return tests
89
90         tests = []
91         for test_dir in self.TEST_DIRS:
92             absolute_test_dir = os.path.join(self._programs_path, test_dir)
93             if not os.path.isdir(absolute_test_dir):
94                 continue
95             for test_file in os.listdir(absolute_test_dir):
96                 if not test_file.lower().startswith("test"):
97                     continue
98                 test_path = os.path.join(self._programs_path, test_dir, test_file)
99                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
100                     tests.append(test_path)
101         return tests
102
103     def _lookup_atspi2_binary(self, filename):
104         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
105         if not exec_prefix:
106             return None
107         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
108             filepath = os.path.join(exec_prefix, path, filename)
109             if os.path.isfile(filepath):
110                 return filepath
111
112         return None
113
114     def _start_accessibility_daemons(self):
115         return False
116         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
117         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
118         if not spi_bus_launcher_path or not spi_registryd_path:
119             return False
120
121         try:
122             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
123         except:
124             sys.stderr.write("Failed to launch the accessibility bus\n")
125             sys.stderr.flush()
126             return False
127
128         # We need to wait until the SPI bus is launched before trying to start the SPI
129         # registry, so we spin a main loop until the bus name appears on DBus.
130         loop = GLib.MainLoop()
131         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
132                            lambda *args: loop.quit(), None)
133         loop.run()
134
135         try:
136             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
137         except:
138             sys.stderr.write("Failed to launch the accessibility registry\n")
139             sys.stderr.flush()
140             return False
141
142         return True
143
144     def _setup_testing_environment(self):
145         self._test_env = os.environ
146         self._test_env["DISPLAY"] = self._options.display
147         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
148         self._test_env['GSETTINGS_BACKEND'] = 'memory'
149         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
150         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path("Libraries")
151         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
152
153         try:
154             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
155                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
156         except Exception as e:
157             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
158             sys.stderr.flush()
159             return False
160
161         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
162         if not self._start_accessibility_daemons():
163             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
164             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
165         return True
166
167     def _tear_down_testing_environment(self):
168         if self._spi_registryd:
169             self._spi_registryd.terminate()
170         if self._spi_bus_launcher:
171             self._spi_bus_launcher.terminate()
172         self._xvfb.terminate()
173
174     def _test_cases_to_skip(self, test_program):
175         if self._options.skipped_action != 'skip':
176             return []
177
178         test_cases = []
179         for skipped in self._skipped_tests:
180             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
181                 test_cases.append(skipped.test_case)
182         return test_cases
183
184     def _should_run_test_program(self, test_program):
185         # This is not affected by the command-line arguments, since programs are skipped for
186         # problems in the harness, such as failing to start the accessibility bus.
187         for skipped in self._skipped_tests:
188             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
189                 return False
190         return True
191
192     def _get_child_pid_from_test_output(self, output):
193         if not output:
194             return -1
195         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
196         if not match:
197             return -1
198         return int(match.group('child_pid'))
199
200     def _kill_process(self, pid):
201         try:
202             os.kill(pid, SIGKILL)
203         except OSError:
204             # Process already died.
205             pass
206
207     def _run_test_command(self, command, timeout=-1):
208         def alarm_handler(signum, frame):
209             raise TestTimeout
210
211         p = subprocess.Popen(command, stdout=subprocess.PIPE, env=self._test_env)
212         if timeout > 0:
213             signal(SIGALRM, alarm_handler)
214             alarm(timeout)
215
216         stdout = ""
217         try:
218             stdout = p.communicate()[0]
219             if timeout > 0:
220                 alarm(0)
221             sys.stdout.write(stdout)
222             sys.stdout.flush()
223         except TestTimeout:
224             self._kill_process(p.pid)
225             child_pid = self._get_child_pid_from_test_output(stdout)
226             if child_pid > 0:
227                 self._kill_process(child_pid)
228             raise
229
230         return not p.returncode
231
232     def _run_test_glib(self, test_program):
233         tester_command = ['gtester']
234         if self._options.verbose:
235             tester_command.append('--verbose')
236         for test_case in self._test_cases_to_skip(test_program):
237             tester_command.extend(['-s', test_case])
238         tester_command.append(test_program)
239
240         return self._run_test_command(tester_command, self._options.timeout)
241
242     def _run_test_google(self, test_program):
243         tester_command = [test_program]
244         skipped_tests_cases = self._test_cases_to_skip(test_program)
245         if skipped_tests_cases:
246             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
247
248         return self._run_test_command(tester_command, self._options.timeout)
249
250     def _run_test(self, test_program):
251         if "unittests" in test_program or "WebKit2APITests" in test_program:
252             return self._run_test_glib(test_program)
253
254         if "TestWebKitAPI" in test_program:
255             return self._run_test_google(test_program)
256
257         return False
258
259     def run_tests(self):
260         if not self._tests:
261             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
262             sys.stderr.flush()
263             return 1
264
265         if not self._setup_testing_environment():
266             return 1
267
268         # Remove skipped tests now instead of when we find them, because
269         # some tests might be skipped while setting up the test environment.
270         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
271
272         failed_tests = []
273         timed_out_tests = []
274         try:
275             for test in self._tests:
276                 success = True
277                 try:
278                     success = self._run_test(test)
279                 except TestTimeout:
280                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
281                     sys.stdout.flush()
282                     timed_out_tests.append(test)
283
284                 if not success:
285                     failed_tests.append(test)
286         finally:
287             self._tear_down_testing_environment()
288
289         if failed_tests:
290             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
291             sys.stdout.write("Tests failed: %s\n" % ", ".join(names))
292             sys.stdout.flush()
293
294         if timed_out_tests:
295             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
296             sys.stdout.write("Tests that timed out: %s\n" % ", ".join(names))
297             sys.stdout.flush()
298
299         if self._skipped_tests and self._options.skipped_action == 'skip':
300             sys.stdout.write("Tests skipped:\n%s\n" % "\n".join([str(skipped) for skipped in self._skipped_tests]))
301             sys.stdout.flush()
302
303         return len(failed_tests)
304
305 if __name__ == "__main__":
306     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
307     option_parser.add_option('-r', '--release',
308                              action='store_true', dest='release',
309                              help='Run in Release')
310     option_parser.add_option('-d', '--debug',
311                              action='store_true', dest='debug',
312                              help='Run in Debug')
313     option_parser.add_option('-v', '--verbose',
314                              action='store_true', dest='verbose',
315                              help='Run gtester in verbose mode')
316     option_parser.add_option('--display', action='store', dest='display', default=':55',
317                              help='Display to run Xvfb')
318     option_parser.add_option('--skipped', action='store', dest='skipped_action',
319                              choices=['skip', 'ignore', 'only'], default='skip',
320                              metavar='skip|ignore|only',
321                              help='Specifies how to treat the skipped tests')
322     option_parser.add_option('-t', '--timeout',
323                              action='store', type='int', dest='timeout', default=10,
324                              help='Time in seconds until a test times out')
325     options, args = option_parser.parse_args()
326
327     sys.exit(TestRunner(options, args).run_tests())